Skip to content

The only tool you need to know what is happening and how to fix it.

progress

dduxe173a1ffce4e
diff --git a/backend/.gitignore b/backend/.gitignore
new file mode 100644
index 00000000..d51c560a
--- /dev/null
+++ b/backend/.gitignore
@@ -0,0 +1 @@
+PROJECT_SUMMARY.md
diff --git a/backend/app/datawarehouse/datawarehouse.go b/backend/app/chdb/chdb.go
similarity index 94%
rename from backend/app/datawarehouse/datawarehouse.go
rename to backend/app/chdb/chdb.go
index 80e2a1ea..2b3b0943 100644
--- a/backend/app/datawarehouse/datawarehouse.go
+++ b/backend/app/chdb/chdb.go
@@ -1,4 +1,4 @@
-package datawarehouse
+package chdb
import (
"crypto/tls"
@@ -21,8 +21,9 @@ func Init() error {
clickhouseDatabase := os.Getenv("CLICKHOUSE_DATABASE")
clickhouseUsername := os.Getenv("CLICKHOUSE_USERNAME")
clickhousePassword := os.Getenv("CLICKHOUSE_PASSWORD")
+ clickhouseTls := os.Getenv("CLICKHOUSE_TLS")
- if strings.HasSuffix(clickhouseServer, "9000") {
+ if clickhouseTls == "false" {
tlsConfig = nil
}
diff --git a/backend/app/controllers/client.controller.go b/backend/app/controllers/client.controller.go
deleted file mode 100644
index 0fbbe7e4..00000000
--- a/backend/app/controllers/client.controller.go
+++ /dev/null
@@ -1,42 +0,0 @@
-package controllers
-
-import (
- "fmt"
- "net/http"
-
- "github.com/gin-gonic/gin"
-)
-
-type clientController struct{}
-
-func (e clientController) Report(c *gin.Context) {
- // we need to parse the request
- // var request []*clientmodels.CollectionFrame
- // if err := c.ShouldBindBodyWithJSON(&request); err != nil {
- // c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
- // return
- // }
-
- b := []byte{}
-
- c.ShouldBindBodyWithPlain(&b)
-
- fmt.Println(string(b))
-
- // tx := middleware.GetTx(c)
- // for _, frame := range request {
- // frame.Metrics
- // frame.StackTraces
- // frame.Transactions
- // }
- // exceptions, err := repositories.ExceptionRepository.FindAll(tx)
-
- // if err != nil {
- // c.AbortWithStatus(http.StatusInternalServerError)
- // panic(err)
- // }
-
- c.JSON(http.StatusOK, gin.H{})
-}
-
-var ClientController = clientController{}
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
new file mode 100644
index 00000000..dec24033
--- /dev/null
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -0,0 +1,124 @@
+package clientcontrollers
+
+import (
+ "backend/app/models"
+ "backend/app/models/clientmodels"
+ "backend/app/repositories"
+ "crypto/sha256"
+ "encoding/hex"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+)
+
+type clientController struct{}
+
+type ReportRequest struct {
+ App string `json:"app"`
+ CollectionFrames []*clientmodels.CollectionFrame `json:"collectionFrames"`
+}
+
+func (e clientController) Report(c *gin.Context) {
+ // we need to parse the request
+ var request ReportRequest
+ if err := c.ShouldBindBodyWithJSON(&request); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ transactionsToInsert := []models.Transaction{}
+ exceptionStackTraceToInsert := []models.ExceptionStackTrace{}
+ metricRecordsToInsert := []models.MetricRecord{}
+ for _, cf := range request.CollectionFrames {
+ for _, ct := range cf.Transactions {
+ transactionsToInsert = append(transactionsToInsert, ct.ToTransaction())
+ }
+
+ for _, cst := range cf.StackTraces {
+ exceptionStackTraceToInsert = append(exceptionStackTraceToInsert, cst.ToExceptionStackTrace(computeExceptionHash(cst.StackTrace)))
+ }
+
+ for _, cm := range cf.Metrics {
+ metricRecordsToInsert = append(metricRecordsToInsert, cm.ToMetricRecord())
+ }
+ }
+
+ err := repositories.TransactionRepository.InsertAsync(c, transactionsToInsert)
+
+ if err != nil {
+ panic(err)
+ }
+
+ err = repositories.ExceptionStackTraceRepository.InsertAsync(c, exceptionStackTraceToInsert)
+
+ if err != nil {
+ panic(err)
+ }
+
+ err = repositories.MetricRecordRepository.InsertAsync(c, metricRecordsToInsert)
+
+ if err != nil {
+ panic(err)
+ }
+
+ c.JSON(http.StatusOK, gin.H{})
+}
+
+var (
+ errorMessageRe = regexp.MustCompile(`(?m)^(\*?[\w.]+):\s*.+`)
+ absolutePathRe = regexp.MustCompile(`/[^\s:]+/([^/\s:]+:\d+)`)
+ versionRe = regexp.MustCompile(`@v[\d.]+`)
+ hexRe = regexp.MustCompile(`0x[0-9a-fA-F]+`)
+ uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`)
+ largeNumberRe = regexp.MustCompile(`(^|[^:\d])(\d{5,})($|[^\d])`)
+ emailRe = regexp.MustCompile(`[\w.\-]+@[\w.\-]+\.\w+`)
+ ipRe = regexp.MustCompile(`\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?`)
+ goroutineRe = regexp.MustCompile(`goroutine \d+`)
+ spacesRe = regexp.MustCompile(`[ \t]+`)
+ newlinesRe = regexp.MustCompile(`\n+`)
+)
+
+func computeExceptionHash(stackTrace string) string {
+ normalized := stackTrace
+
+ // Remove the error message content (keep just the error type)
+ normalized = errorMessageRe.ReplaceAllString(normalized, "$1")
+
+ // Remove absolute paths, keep just filename:line
+ normalized = absolutePathRe.ReplaceAllString(normalized, "$1")
+
+ // Remove version numbers from module paths
+ normalized = versionRe.ReplaceAllString(normalized, "")
+
+ // Replace hex addresses/pointers
+ normalized = hexRe.ReplaceAllString(normalized, "<hex>")
+
+ // Replace UUIDs
+ normalized = uuidRe.ReplaceAllString(normalized, "<uuid>")
+
+ // Replace standalone large numbers (likely IDs, not line numbers)
+ // Since Go doesn't support lookbehind, we preserve the surrounding characters
+ normalized = largeNumberRe.ReplaceAllString(normalized, "${1}<id>${3}")
+
+ // Replace email addresses
+ normalized = emailRe.ReplaceAllString(normalized, "<email>")
+
+ // Replace IP addresses
+ normalized = ipRe.ReplaceAllString(normalized, "<ip>")
+
+ // Normalize goroutine numbers
+ normalized = goroutineRe.ReplaceAllString(normalized, "goroutine <n>")
+
+ // Normalize whitespace
+ normalized = spacesRe.ReplaceAllString(normalized, " ")
+ normalized = newlinesRe.ReplaceAllString(normalized, "\n")
+
+ // Compute SHA-256 and return first 16 hex characters
+ normalized = strings.TrimSpace(normalized)
+ hash := sha256.Sum256([]byte(normalized))
+ return hex.EncodeToString(hash[:])[:16]
+}
+
+var ClientController = clientController{}
diff --git a/backend/app/controllers/exception.controller.go b/backend/app/controllers/exception.controller.go
deleted file mode 100644
index 672f3024..00000000
--- a/backend/app/controllers/exception.controller.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package controllers
-
-type exceptionController struct{}
-
-// func (e exceptionController) FindAll(c *gin.Context) {
-// tx := middleware.GetTx(c)
-// exceptions, err := repositories.ExceptionRepository.FindAll(tx)
-
-// if err != nil {
-// c.AbortWithStatus(http.StatusInternalServerError)
-// panic(err)
-// }
-
-// c.JSON(http.StatusOK, gin.H{
-// "exceptions": exceptions,
-// })
-// }
-
-var ExceptionController = exceptionController{}
diff --git a/backend/app/controllers/metric_record.controller.go b/backend/app/controllers/metric_record.controller.go
new file mode 100644
index 00000000..976f3007
--- /dev/null
+++ b/backend/app/controllers/metric_record.controller.go
@@ -0,0 +1,17 @@
+package controllers
+
+import (
+ "github.com/gin-gonic/gin"
+)
+
+type metricRecordController struct{}
+
+func (e metricRecordController) FindHomepageStats(c *gin.Context) {
+ // requests in last 24h vs previous 24h
+ // exceptions in last 24h vs previous 24h
+ // ram usage last 24h vs previous 24h
+ // memory usage last 24h vs previous 24h
+
+}
+
+var MetricRecordController = metricRecordController{}
diff --git a/backend/app/controllers/routes.go b/backend/app/controllers/routes.go
index 569b983e..3bbfd320 100644
--- a/backend/app/controllers/routes.go
+++ b/backend/app/controllers/routes.go
@@ -1,11 +1,34 @@
package controllers
import (
+ "backend/app/controllers/clientcontrollers"
"backend/app/middleware"
"github.com/gin-gonic/gin"
)
+type PaginationParams struct {
+ Page int `form:"page" binding:"min=1"`
+ PageSize int `form:"page_size" binding:"min=1,max=100"`
+}
+
+type PaginatedResponse[T any] struct {
+ Data []T `json:"data"`
+ Pagination Pagination `json:"pagination"`
+}
+
+type Pagination struct {
+ Page int `json:"page"`
+ PageSize int `json:"page_size"`
+ Total int64 `json:"total"`
+ TotalPages int64 `json:"total_pages"`
+}
+
func RegisterControllers(router *gin.RouterGroup) {
- router.POST("/report", middleware.UseAuth, middleware.UseGzip, middleware.UseTransaction, ClientController.Report)
+ router.POST("/report", middleware.UseClientAuth, middleware.UseGzip, clientcontrollers.ClientController.Report)
+
+ router.POST("/stats", middleware.UseAppAuth, middleware.UseGzip, MetricRecordController.FindHomepageStats)
+
+ // router.POST("/transactions", middleware.UseAppAuth, middleware.UseGzip, TransactionController.FindAllTransactions)
+ // router.POST("/exception-stack-traces", middleware.UseAppAuth, middleware.UseGzip, AppController.FindAllExceptionStackTraces)
}
diff --git a/backend/app/db/db.go b/backend/app/db/db.go
deleted file mode 100644
index c4c0ecef..00000000
--- a/backend/app/db/db.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package db
-
-import (
- "database/sql"
- "log"
- "os"
-)
-
-var DB *sql.DB
-
-func Init() {
- db, err := sql.Open("pgx", os.Getenv("DB"))
- if err != nil {
- log.Fatalf("Unable to connect to database: %v\n", err)
- }
-
- DB = db
-}
diff --git a/backend/app/middleware/use_auth.middleware.go b/backend/app/middleware/use_app_auth.middleware copy.go
similarity index 72%
rename from backend/app/middleware/use_auth.middleware.go
rename to backend/app/middleware/use_app_auth.middleware copy.go
index b588f380..c9c28d04 100644
--- a/backend/app/middleware/use_auth.middleware.go
+++ b/backend/app/middleware/use_app_auth.middleware copy.go
@@ -7,12 +7,12 @@ import (
"github.com/gin-gonic/gin"
)
-var UseAuth func(c *gin.Context)
+var UseAppAuth func(c *gin.Context)
-func InitUseAuth() {
+func InitUseAppAuth() {
authHeader := "Bearer " + os.Getenv("TOKEN")
- UseAuth = func(c *gin.Context) {
+ UseAppAuth = func(c *gin.Context) {
if authHeader != c.GetHeader("Authorization") {
c.AbortWithStatus(http.StatusUnauthorized)
return
diff --git a/backend/app/middleware/use_client_auth.middleware.go b/backend/app/middleware/use_client_auth.middleware.go
new file mode 100644
index 00000000..0cd78245
--- /dev/null
+++ b/backend/app/middleware/use_client_auth.middleware.go
@@ -0,0 +1,23 @@
+package middleware
+
+import (
+ "net/http"
+ "os"
+
+ "github.com/gin-gonic/gin"
+)
+
+var UseClientAuth func(c *gin.Context)
+
+func InitUseClientAuth() {
+ authHeader := "Bearer " + os.Getenv("TOKEN")
+
+ UseClientAuth = func(c *gin.Context) {
+ if authHeader != c.GetHeader("Authorization") {
+ c.AbortWithStatus(http.StatusUnauthorized)
+ return
+ }
+
+ c.Next()
+ }
+}
diff --git a/backend/app/middleware/use_transaction.middleware.go b/backend/app/middleware/use_transaction.middleware.go
deleted file mode 100644
index f097af6f..00000000
--- a/backend/app/middleware/use_transaction.middleware.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package middleware
-
-import (
- "backend/app/db"
- "database/sql"
- "fmt"
- "net/http"
-
- "github.com/gin-gonic/gin"
-)
-
-func GetTx(c *gin.Context) *sql.Tx {
- // this will panic if we call it in a non transactional route - we're ok with that
- txFromC, exists := c.Get("tx")
- if !exists {
- panic("No TX bound to the context, please make sure you're using the UseTransaction middleware")
- }
-
- tx, ok := txFromC.(*sql.Tx)
-
- if !ok {
- panic("No TX bound to the context, please make sure you're using the UseTransaction middleware")
- }
-
- return tx
-}
-
-// only commits the tx for status 200 - ignores 201
-func UseTransaction(c *gin.Context) {
- tx, err := db.DB.Begin()
- if err != nil {
- _ = c.Error(fmt.Errorf("failed to begin transaction: %w", err))
- c.Abort()
- return
- }
-
- defer func() {
- if p := recover(); p != nil {
- tx.Rollback()
- panic(p)
- } else if status := c.Writer.Status(); status == http.StatusOK || status == http.StatusCreated {
- if err := tx.Commit(); err != nil {
- _ = c.Error(fmt.Errorf("failed to commit: %w", err))
- tx.Rollback()
- }
- } else {
- tx.Rollback()
- }
- }()
-
- c.Set("tx", tx)
- c.Next()
-}
diff --git a/backend/app/migrations/ch/0001_init.up.sql b/backend/app/migrations/ch/0001_init.up.sql
deleted file mode 100644
index 6d87862d..00000000
--- a/backend/app/migrations/ch/0001_init.up.sql
+++ /dev/null
@@ -1,19 +0,0 @@
-CREATE TABLE demos
-(
- `device_id` UInt32,
- `organization_id` UInt32,
- `safety_trip_id` UUID,
- `event_type` UInt8,
- `event_time` DateTime64(3, 'UTC'),
- `pos_lat` Float64,
- `pos_long` Float64,
- `odometer` Nullable(Float64),
- `velocity` Nullable(Float32),
- `location` String,
- `total_miles` Nullable(Float32),
- `total_score` Nullable(UInt32)
-)
-ENGINE = MergeTree
-PARTITION BY toYYYYMM(event_time)
-ORDER BY (organization_id, device_id, safety_trip_id, event_time)
-SETTINGS index_granularity = 8192
\ No newline at end of file
diff --git a/backend/app/migrations/migrations.go b/backend/app/migrations/migrations.go
deleted file mode 100644
index d0d923e3..00000000
--- a/backend/app/migrations/migrations.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package migrations
-
-import (
- "backend/app/db"
- "database/sql"
- "embed"
- "fmt"
- "os"
-
- "github.com/golang-migrate/migrate/v4"
- migrateCh "github.com/golang-migrate/migrate/v4/database/clickhouse"
- "github.com/golang-migrate/migrate/v4/database/postgres"
- "github.com/golang-migrate/migrate/v4/source/iofs"
-)
-
-//go:embed pg/*.sql
-var migrationsPgFS embed.FS
-
-//go:embed ch/*.sql
-var migrationsChFS embed.FS
-
-func runMigrationsPg(db *sql.DB) error {
- source, err := iofs.New(migrationsPgFS, "pg")
- if err != nil {
- return err
- }
-
- // Create database driver from existing connection
- driver, err := postgres.WithInstance(db, &postgres.Config{})
- if err != nil {
- return err
- }
-
- // Create migrator with both source and database instances
- m, err := migrate.NewWithInstance("iofs", source, "postgres", driver)
- if err != nil {
- return err
- }
-
- // Run migrations (ignore ErrNoChange)
- if err := m.Up(); err != nil && err != migrate.ErrNoChange {
- return err
- }
-
- return nil
-}
-
-func runMigrationsClickhouse(connStr string) error {
- db, err := sql.Open("clickhouse", connStr)
- if err != nil {
- return err
- }
- defer db.Close()
-
- source, err := iofs.New(migrationsChFS, "ch")
- if err != nil {
- return err
- }
-
- driver, err := migrateCh.WithInstance(db, &migrateCh.Config{})
- if err != nil {
- return err
- }
-
- m, err := migrate.NewWithInstance("iofs", source, "clickhouse", driver)
- if err != nil {
- return err
- }
-
- if err := m.Up(); err != nil && err != migrate.ErrNoChange {
- return err
- }
-
- return nil
-}
-
-func Run() {
- err := runMigrationsPg(db.DB)
-
- if err != nil {
- panic(err)
- }
-
- clickhouseServer := os.Getenv("CLICKHOUSE_SERVER")
- clickhouseDatabase := os.Getenv("CLICKHOUSE_DATABASE")
- clickhouseUsername := os.Getenv("CLICKHOUSE_USERNAME")
- clickhousePassword := os.Getenv("CLICKHOUSE_PASSWORD")
-
- err = runMigrationsClickhouse(fmt.Sprintf(`clickhouse://%s?username=%s&password=%s&database=%s`, clickhouseServer, clickhouseUsername, clickhousePassword, clickhouseDatabase))
-
- if err != nil {
- panic(err)
- }
-}
diff --git a/backend/app/migrations/pg/0001_init.up.sql b/backend/app/migrations/pg/0001_init.up.sql
deleted file mode 100644
index a3733730..00000000
--- a/backend/app/migrations/pg/0001_init.up.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-DROP TABLE users;
-CREATE TABLE IF NOT EXISTS users (
- id SERIAL PRIMARY KEY,
- first_name TEXT NOT NULL,
- last_name TEXT NOT NULL,
- email TEXT UNIQUE NOT NULL
-);
\ No newline at end of file
diff --git a/backend/app/models/clientmodels/clientmodels.model.go b/backend/app/models/clientmodels/clientmodels.model.go
index bcd5cb76..414511a6 100644
--- a/backend/app/models/clientmodels/clientmodels.model.go
+++ b/backend/app/models/clientmodels/clientmodels.model.go
@@ -1,6 +1,9 @@
package clientmodels
-import "time"
+import (
+ "backend/app/models"
+ "time"
+)
type ClientExceptionStackTrace struct {
TransactionId *string `json:"transactionId"`
@@ -8,12 +11,29 @@ type ClientExceptionStackTrace struct {
RecordedAt time.Time `json:"recordedAt"`
}
-type ClientMetricsRecord struct {
+func (c *ClientExceptionStackTrace) ToExceptionStackTrace(exceptionHash string) models.ExceptionStackTrace {
+ return models.ExceptionStackTrace{
+ ExceptionHash: exceptionHash,
+ TransactionId: c.TransactionId,
+ StackTrace: c.StackTrace,
+ RecordedAt: c.RecordedAt,
+ }
+}
+
+type ClientMetricRecord struct {
Name string `json:"name"`
Value float32 `json:"value"`
RecordedAt time.Time `json:"recordedAt"`
}
+func (c *ClientMetricRecord) ToMetricRecord() models.MetricRecord {
+ return models.MetricRecord{
+ Name: c.Name,
+ Value: c.Value,
+ RecordedAt: c.RecordedAt,
+ }
+}
+
type ClientTransaction struct {
Id string `json:"id"`
Endpoint string `json:"endpoint"`
@@ -24,8 +44,20 @@ type ClientTransaction struct {
ClientIP string `json:"clientIP"`
}
+func (c *ClientTransaction) ToTransaction() models.Transaction {
+ return models.Transaction{
+ Id: c.Id,
+ Endpoint: c.Endpoint,
+ Duration: c.Duration,
+ RecordedAt: c.RecordedAt,
+ StatusCode: c.StatusCode,
+ BodySize: c.BodySize,
+ ClientIP: c.ClientIP,
+ }
+}
+
type CollectionFrame struct {
StackTraces []*ClientExceptionStackTrace `json:"stackTraces"`
- Metrics []*ClientMetricsRecord `json:"metrics"`
+ Metrics []*ClientMetricRecord `json:"metrics"`
Transactions []*ClientTransaction `json:"transactions"`
}
diff --git a/backend/app/models/exception.model.go b/backend/app/models/exception.model.go
deleted file mode 100644
index 64a331b0..00000000
--- a/backend/app/models/exception.model.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package models
-
-import (
- "time"
-
- "github.com/tracewayapp/go-lightning/lpg"
-)
-
-type Exception struct {
- Id int
- Archived bool
- FirstSeen time.Time
- LastSeen time.Time
- Title string
-}
-
-func init() {
- lpg.Register[Exception]()
-}
diff --git a/backend/app/models/exception_stack_trace.model.go b/backend/app/models/exception_stack_trace.model.go
index 23104669..5be0c612 100644
--- a/backend/app/models/exception_stack_trace.model.go
+++ b/backend/app/models/exception_stack_trace.model.go
@@ -4,6 +4,7 @@ import "time"
type ExceptionStackTrace struct {
TransactionId *string `json:"transactionId" ch:"transaction_id"`
+ ExceptionHash string `json:"exceptionHash" ch:"exception_hash"`
StackTrace string `json:"stackTrace" ch:"stack_trace"`
RecordedAt time.Time `json:"recordedAt" ch:"recorded_at"`
}
diff --git a/backend/app/models/metric.model.go b/backend/app/models/metric_record.model.go
similarity index 86%
rename from backend/app/models/metric.model.go
rename to backend/app/models/metric_record.model.go
index 5de9e822..51c47e9e 100644
--- a/backend/app/models/metric.model.go
+++ b/backend/app/models/metric_record.model.go
@@ -2,7 +2,7 @@ package models
import "time"
-type MetricsRecord struct {
+type MetricRecord struct {
Name string `json:"name" ch:"name"`
Value float32 `json:"value" ch:"value"`
RecordedAt time.Time `json:"recordedAt" ch:"recorded_at"`
diff --git a/backend/app/models/transaction.model.go b/backend/app/models/transaction.model.go
index d3c469bb..0eb142f8 100644
--- a/backend/app/models/transaction.model.go
+++ b/backend/app/models/transaction.model.go
@@ -3,7 +3,8 @@ package models
import "time"
type Transaction struct {
- Id string `json:"id" ch:"id"`
+ Id string `json:"id" ch:"id"`
+ // endpoint is the route from the router/does not contain actual params so it's safe to group on it
Endpoint string `json:"endpoint" ch:"endpoint"`
Duration time.Duration `json:"duration" ch:"duration"`
RecordedAt time.Time `json:"recordedAt" ch:"recorded_at"`
diff --git a/backend/app/repositories/exception_stack_trace.repository.go b/backend/app/repositories/exception_stack_trace.repository.go
new file mode 100644
index 00000000..9a969d58
--- /dev/null
+++ b/backend/app/repositories/exception_stack_trace.repository.go
@@ -0,0 +1,24 @@
+package repositories
+
+import (
+ "backend/app/chdb"
+ "backend/app/models"
+ "context"
+)
+
+type exceptionStackTraceRepository struct{}
+
+func (e *exceptionStackTraceRepository) InsertAsync(ctx context.Context, lines []models.ExceptionStackTrace) error {
+ batch, err := (*chdb.Conn).PrepareBatch(ctx, "INSERT INTO exception_stack_traces (transaction_id, exception_hash, stack_trace, recorded_at)")
+ if err != nil {
+ return err
+ }
+ for _, e := range lines {
+ if err := batch.Append(e.TransactionId, e.ExceptionHash, e.StackTrace, e.RecordedAt); err != nil {
+ return err
+ }
+ }
+ return batch.Send()
+}
+
+var ExceptionStackTraceRepository = exceptionStackTraceRepository{}
diff --git a/backend/app/repositories/metric_record.repository.go b/backend/app/repositories/metric_record.repository.go
new file mode 100644
index 00000000..511a4ee1
--- /dev/null
+++ b/backend/app/repositories/metric_record.repository.go
@@ -0,0 +1,24 @@
+package repositories
+
+import (
+ "backend/app/chdb"
+ "backend/app/models"
+ "context"
+)
+
+type metricRecordRepository struct{}
+
+func (e *metricRecordRepository) InsertAsync(ctx context.Context, lines []models.MetricRecord) error {
+ batch, err := (*chdb.Conn).PrepareBatch(ctx, "INSERT INTO metric_records (name, value, recorded_at)")
+ if err != nil {
+ return err
+ }
+ for _, e := range lines {
+ if err := batch.Append(e.Name, e.Value, e.RecordedAt); err != nil {
+ return err
+ }
+ }
+ return batch.Send()
+}
+
+var MetricRecordRepository = metricRecordRepository{}
diff --git a/backend/app/repositories/transaction.repository.go b/backend/app/repositories/transaction.repository.go
index cf2c215d..da60dc5d 100644
--- a/backend/app/repositories/transaction.repository.go
+++ b/backend/app/repositories/transaction.repository.go
@@ -1,20 +1,24 @@
package repositories
import (
+ "backend/app/chdb"
"backend/app/models"
- "database/sql"
-
- "github.com/tracewayapp/go-lightning/lpg"
+ "context"
)
type transactionRepository struct{}
-func (e transactionRepository) FindAll(tx *sql.Tx) ([]*models.Transaction, error) {
- return lpg.SelectGeneric[models.Transaction](tx, "SELECT * FROM transactions")
+func (e *transactionRepository) InsertAsync(ctx context.Context, lines []models.Transaction) error {
+ batch, err := (*chdb.Conn).PrepareBatch(ctx, "INSERT INTO transactions (id, endpoint, duration, recorded_at, status_code, body_size, client_ip)")
+ if err != nil {
+ return err
+ }
+ for _, e := range lines {
+ if err := batch.Append(e.Id, e.Endpoint, e.Duration, e.RecordedAt, e.StatusCode, e.BodySize, e.ClientIP); err != nil {
+ return err
+ }
+ }
+ return batch.Send()
}
-// func (e transactionRepository) Create(tx *sql.Tx, transaction *models.Transaction) error {
-// return lpg.Create[models.Transaction](tx, "SELECT * FROM transactions")
-// }
-
var TransactionRepository = transactionRepository{}
diff --git a/backend/main.go b/backend/main.go
index 8c9b714b..b1a38a11 100644
--- a/backend/main.go
+++ b/backend/main.go
@@ -1,12 +1,12 @@
package main
import (
+ "backend/app/chdb"
"backend/app/controllers"
"backend/app/middleware"
"log"
"github.com/gin-gonic/gin"
- _ "github.com/jackc/pgx/v5/stdlib"
"github.com/joho/godotenv"
)
@@ -16,12 +16,9 @@ func main() {
log.Fatal("Error loading .env file")
}
- // db.Init()
- // datawarehouse.Init()
+ chdb.Init()
- // migrations.Run()
-
- middleware.InitUseAuth()
+ middleware.InitUseClientAuth()
router := gin.Default()
diff --git a/clients/go-client/devtesting/devtesting.go b/clients/go-client/devtesting/devtesting.go
index 8d0e1e1d..ab372770 100644
--- a/clients/go-client/devtesting/devtesting.go
+++ b/clients/go-client/devtesting/devtesting.go
@@ -14,7 +14,11 @@ func main() {
func testGin() {
router := gin.Default()
- router.Use(tracewaygin.New("demotoken@localhost:8081/report"))
+ router.Use(tracewaygin.New(
+ "tracewaydemo",
+ "demotoken@http://localhost:8082/api/report",
+ traceway.WithDebug(true),
+ ))
router.GET("/test-exception", func(ctx *gin.Context) {
panic("Cool")
diff --git a/clients/go-client/traceway.go b/clients/go-client/traceway.go
index 459a7771..8f39aeec 100644
--- a/clients/go-client/traceway.go
+++ b/clients/go-client/traceway.go
@@ -2,6 +2,7 @@ package traceway
import (
"bytes"
+ "compress/gzip"
"encoding/json"
"fmt"
"log"
@@ -12,6 +13,8 @@ import (
"strings"
"sync"
"time"
+
+ "github.com/gin-gonic/gin"
)
func CaptureStack(skip int) []runtime.Frame {
@@ -130,6 +133,7 @@ type CollectionFrameStore struct {
lastUploadStarted *time.Time
// Config fields
+ app string
apiUrl string
token string
debug bool
@@ -139,6 +143,7 @@ type CollectionFrameStore struct {
}
func InitCollectionFrameStore(
+ app string,
apiUrl string,
token string,
debug bool,
@@ -153,6 +158,7 @@ func InitCollectionFrameStore(
stopCh: make(chan struct{}),
messageQueue: make(chan CollectionFrameMessage),
+ app: app,
apiUrl: apiUrl,
token: token,
debug: debug,
@@ -234,16 +240,35 @@ func (s *CollectionFrameStore) triggerUpload(framesToSend []*CollectionFrame) {
}
}()
- jsonData, err := json.Marshal(framesToSend)
+ jsonData, err := json.Marshal(gin.H{
+ "app": s.app,
+ "frames": framesToSend,
+ })
if err != nil {
if s.debug {
log.Printf("Traceway: failed to marshal frames: %v", err)
}
return
}
+
fmt.Println("SENDING POST TO ", s.apiUrl, string(jsonData))
- req, err := http.NewRequest("POST", s.apiUrl, bytes.NewBuffer(jsonData))
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ if _, err := gz.Write(jsonData); err != nil {
+ if s.debug {
+ log.Printf("Traceway: gz write failed: %v", err)
+ }
+ return
+ }
+ if err := gz.Close(); err != nil { // Close flushes the compressed data
+ if s.debug {
+ log.Printf("Traceway: gz write failed: %v", err)
+ }
+ return
+ }
+
+ req, err := http.NewRequest("POST", s.apiUrl, &buf)
if err != nil {
if s.debug {
log.Printf("Traceway: failed to create request: %v", err)
@@ -253,6 +278,7 @@ func (s *CollectionFrameStore) triggerUpload(framesToSend []*CollectionFrame) {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+s.token)
+ req.Header.Set("Content-Encoding", "gzip")
client := &http.Client{}
resp, err := client.Do(req)
@@ -293,7 +319,7 @@ type TracewayOptions struct {
func NewTracewayOptions(options ...func(*TracewayOptions)) *TracewayOptions {
svr := &TracewayOptions{
- maxCollectionFrames: 5,
+ maxCollectionFrames: 12,
collectionInterval: 5 * time.Second,
uploadTimeout: 2 * time.Second,
}
@@ -323,7 +349,7 @@ func WithUploadTimeout(val time.Duration) func(*TracewayOptions) {
}
}
-func Init(connectionString string, options ...func(*TracewayOptions)) error {
+func Init(app, connectionString string, options ...func(*TracewayOptions)) error {
if collectionFrameStore != nil {
return fmt.Errorf("Second Traceway initialization detected")
}
@@ -335,6 +361,7 @@ func Init(connectionString string, options ...func(*TracewayOptions)) error {
tracewayOptions := NewTracewayOptions(options...)
collectionFrameStore = InitCollectionFrameStore(
+ app,
apiUrl,
token,
tracewayOptions.debug,
diff --git a/clients/go-client/traceway_gin/traceway_gin.go b/clients/go-client/traceway_gin/traceway_gin.go
index b74664a2..9549542e 100644
--- a/clients/go-client/traceway_gin/traceway_gin.go
+++ b/clients/go-client/traceway_gin/traceway_gin.go
@@ -23,8 +23,8 @@ func wrapAndExecute(c *gin.Context) (s *string) {
return nil
}
-func New(connectionString string, options ...func(*traceway.TracewayOptions)) gin.HandlerFunc {
- traceway.Init(connectionString, options...)
+func New(app, connectionString string, options ...func(*traceway.TracewayOptions)) gin.HandlerFunc {
+ traceway.Init(app, connectionString, options...)
return func(c *gin.Context) {
fmt.Println("INTERCEPTED")