Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Add phase timing for selected database writes

Neeraj60b96246f60d
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go
index 847e9eda81..d90e5176de 100644
--- a/server/pkg/repo/collection.go
+++ b/server/pkg/repo/collection.go
@@ -17,6 +17,7 @@ import (
"github.com/ente/museum/ente"
"github.com/ente/museum/pkg/utils/crypto"
+ "github.com/ente/museum/pkg/utils/dbmetrics"
"github.com/ente/museum/pkg/utils/time"
"github.com/lib/pq"
)
@@ -798,7 +799,9 @@ func (repo *CollectionRepository) AddFiles(
files []ente.CollectionFileItem,
fileOwnerID int64,
) error {
+ finishPhase := dbmetrics.Start(dbmetrics.CollectionAddFiles, "begin_with_checkout")
tx, err := repo.DB.BeginTx(ctx, nil)
+ finishPhase(err)
if err != nil {
return stacktrace.Propagate(err, "")
}
@@ -808,26 +811,38 @@ func (repo *CollectionRepository) AddFiles(
for _, file := range files {
fileIDs = append(fileIDs, file.ID)
}
- if err := lockFiles(ctx, tx, fileOwnerID, fileIDs); err != nil {
+ finishPhase = dbmetrics.Start(dbmetrics.CollectionAddFiles, "lock_files")
+ err = lockFiles(ctx, tx, fileOwnerID, fileIDs)
+ finishPhase(err)
+ if err != nil {
return stacktrace.Propagate(err, "")
}
updationTime := time.Microseconds()
+ finishPhase = dbmetrics.Start(dbmetrics.CollectionAddFiles, "check_trash")
trashedOrDeletedFileIDs, err := repo.TrashRepo.getFilesInTrashOrDeleted(ctx, tx, fileOwnerID, fileIDs)
+ finishPhase(err)
if err != nil {
return stacktrace.Propagate(err, "failed to check trash state")
}
if len(trashedOrDeletedFileIDs) > 0 {
return stacktrace.Propagate(&ente.ErrFileInTrash, "")
}
- if err := upsertCollectionFiles(ctx, tx, collectionID, collectionOwnerID, files, fileOwnerID, updationTime); err != nil {
+ finishPhase = dbmetrics.Start(dbmetrics.CollectionAddFiles, "upsert_memberships")
+ err = upsertCollectionFiles(ctx, tx, collectionID, collectionOwnerID, files, fileOwnerID, updationTime)
+ finishPhase(err)
+ if err != nil {
return stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.CollectionAddFiles, "update_collection")
_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
WHERE collection_id = $2`, updationTime, collectionID)
+ finishPhase(err)
if err != nil {
return stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.CollectionAddFiles, "commit")
err = tx.Commit()
+ finishPhase(err)
return stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/repo/file.go b/server/pkg/repo/file.go
index 0e33122824..50545203fb 100644
--- a/server/pkg/repo/file.go
+++ b/server/pkg/repo/file.go
@@ -14,6 +14,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/ente/museum/ente"
+ "github.com/ente/museum/pkg/utils/dbmetrics"
"github.com/ente/museum/pkg/utils/s3config"
"github.com/ente/museum/pkg/utils/time"
"github.com/lib/pq"
@@ -43,12 +44,15 @@ func (repo *FileRepository) Create(
}
ctx := context.Background()
+ finishPhase := dbmetrics.Start(dbmetrics.FileCreate, "begin_with_checkout")
tx, err := repo.DB.BeginTx(ctx, nil)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
defer tx.Rollback()
var fileID int64
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_file")
err = tx.QueryRowContext(ctx, `INSERT INTO files
(owner_id, app, encrypted_metadata,
file_decryption_header, thumbnail_decryption_header, metadata_decryption_header,
@@ -58,32 +62,41 @@ func (repo *FileRepository) Create(
file.Thumbnail.DecryptionHeader, file.Metadata.DecryptionHeader,
file.MagicMetadata, file.PublicMagicMetadata, file.Info,
file.UpdationTime).Scan(&fileID)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
file.ID = fileID
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_membership")
_, err = tx.ExecContext(ctx, `INSERT INTO collection_files
(collection_id, file_id, encrypted_key, key_decryption_nonce, is_deleted, updation_time, c_owner_id, f_owner_id)
VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, file.CollectionID, file.ID,
file.EncryptedKey, file.KeyDecryptionNonce, false, file.UpdationTime, file.OwnerID, collectionOwnerID)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "update_collection")
_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
WHERE collection_id = $2`, file.UpdationTime, file.CollectionID)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_original_object")
_, err = tx.ExecContext(ctx, `INSERT INTO object_keys(file_id, o_type, object_key, size, datacenters)
VALUES($1, $2, $3, $4, $5)`, fileID, ente.FILE, file.File.ObjectKey, fileSize, dcsForNewEntry)
+ finishPhase(err)
if err != nil {
if err.Error() == "pq: duplicate key value violates unique constraint \"object_keys_object_key_key\"" {
return file, -1, ente.ErrDuplicateFileObjectFound
}
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_thumbnail_object")
_, err = tx.ExecContext(ctx, `INSERT INTO object_keys(file_id, o_type, object_key, size, datacenters)
VALUES($1, $2, $3, $4, $5)`, fileID, ente.THUMBNAIL, file.Thumbnail.ObjectKey, thumbnailSize, dcsForNewEntry)
+ finishPhase(err)
if err != nil {
if err.Error() == "pq: duplicate key value violates unique constraint \"object_keys_object_key_key\"" {
return file, -1, ente.ErrDuplicateThumbnailObjectFound
@@ -91,25 +104,35 @@ func (repo *FileRepository) Create(
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "consume_original_upload")
err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, file.File.ObjectKey, file.OwnerID, app)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "consume_thumbnail_upload")
err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, file.Thumbnail.ObjectKey, file.OwnerID, app)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "update_usage")
usage, err := repo.updateUsageForFileCreation(ctx, tx, file.OwnerID, usageDiff, app)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "mark_replication")
err = repo.markAsNeedingReplication(ctx, tx, file, hotDC)
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
+ finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "commit")
err = tx.Commit()
+ finishPhase(err)
if err != nil {
return file, -1, stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/repo/userentity/data.go b/server/pkg/repo/userentity/data.go
index daf5cd0339..4ec6f82582 100644
--- a/server/pkg/repo/userentity/data.go
+++ b/server/pkg/repo/userentity/data.go
@@ -7,6 +7,7 @@ import (
"github.com/ente/museum/ente"
model "github.com/ente/museum/ente/userentity"
+ "github.com/ente/museum/pkg/utils/dbmetrics"
"github.com/ente/stacktrace"
"github.com/sirupsen/logrus"
)
@@ -107,7 +108,9 @@ func (r *Repository) Update(ctx context.Context, userID int64, req model.UpdateE
query += ` and updated_at = $6`
args = append(args, *req.ExpectedUpdatedAt)
}
+ finishPhase := dbmetrics.Start(dbmetrics.UserEntityUpdate, "exec_autocommit_with_checkout")
result, err := r.DB.ExecContext(ctx, query, args...)
+ finishPhase(err)
if err != nil {
return stacktrace.Propagate(err, "")
}
@@ -116,7 +119,9 @@ func (r *Repository) Update(ctx context.Context, userID int64, req model.UpdateE
return stacktrace.Propagate(err, "")
}
if affected != 1 {
+ finishPhase = dbmetrics.Start(dbmetrics.UserEntityUpdate, "read_conflict_with_checkout")
dbEntity, dbEntityErr := r.Get(ctx, userID, req.ID)
+ finishPhase(dbEntityErr)
if dbEntityErr != nil {
return stacktrace.Propagate(dbEntityErr, "failed to get entity for update with id=%s", req.ID)
}
diff --git a/server/pkg/utils/dbmetrics/latency.go b/server/pkg/utils/dbmetrics/latency.go
new file mode 100644
index 0000000000..7ecd4fc04b
--- /dev/null
+++ b/server/pkg/utils/dbmetrics/latency.go
@@ -0,0 +1,34 @@
+package dbmetrics
+
+import (
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type Operation string
+
+const (
+ CollectionAddFiles Operation = "collection_add_files"
+ FileCreate Operation = "file_create"
+ UserEntityUpdate Operation = "user_entity_update"
+)
+
+var phaseDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "museum_db_phase_duration_seconds",
+ Help: "Elapsed time of selected write-operation phases. Phases ending in _with_checkout include connection acquisition.",
+ Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60},
+}, []string{"operation", "phase", "result"})
+
+func Start(operation Operation, phase string) func(error) {
+ startedAt := time.Now()
+ return func(err error) {
+ elapsed := time.Since(startedAt).Seconds()
+ result := "success"
+ if err != nil {
+ result = "error"
+ }
+ phaseDuration.WithLabelValues(string(operation), phase, result).Observe(elapsed)
+ }
+}