Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: redesign Sheet API with GetSheetMetadata/GetSheetFull (#18499)

Danny Xu3675a417bf40
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.go
index db46d9f9bb..d9e1c9512d 100644
--- a/backend/api/v1/plan_service.go
+++ b/backend/api/v1/plan_service.go
@@ -609,14 +609,11 @@ func (s *PlanService) UpdatePlan(ctx context.Context, request *connect.Request[v
return nil
}
- sheet, err := s.store.GetSheet(ctx, &store.FindSheetMessage{
- UID: &sheetUID,
- ProjectID: &sheetProjectID,
- })
+ sheet, err := s.store.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return connect.NewError(connect.CodeInternal, errors.Errorf("failed to get sheet %q: %v", oldSheetName, err))
}
- if sheet == nil {
+ if sheet.ProjectID != sheetProjectID {
return connect.NewError(connect.CodeNotFound, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID))
}
doUpdate = true
diff --git a/backend/api/v1/plan_service_converter.go b/backend/api/v1/plan_service_converter.go
index 825cd4d392..f0d1b1518e 100644
--- a/backend/api/v1/plan_service_converter.go
+++ b/backend/api/v1/plan_service_converter.go
@@ -298,12 +298,12 @@ func convertToPlanCheckRun(ctx context.Context, s *store.Store, projectID string
}
if sheetUID := int(run.Config.GetSheetUid()); sheetUID != 0 {
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &projectID})
+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet")
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)
+ if sheet.ProjectID != projectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)
}
converted.Sheet = common.FormatSheet(projectID, sheet.UID)
}
diff --git a/backend/api/v1/plan_service_plan_check.go b/backend/api/v1/plan_service_plan_check.go
index a48dfe6eeb..28b07bacbb 100644
--- a/backend/api/v1/plan_service_plan_check.go
+++ b/backend/api/v1/plan_service_plan_check.go
@@ -178,15 +178,12 @@ func getPlanCheckRunsFromChangeDatabaseConfigForDatabase(ctx context.Context, s
if instance == nil {
return nil, errors.Errorf("instance %q not found", database.InstanceID)
}
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{
- UID: &sheetUID,
- ProjectID: &plan.ProjectID,
- })
+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, plan.ProjectID)
+ if sheet.ProjectID != plan.ProjectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, plan.ProjectID)
}
enableSDL := config.Type == storepb.PlanConfig_ChangeDatabaseConfig_SDL
diff --git a/backend/api/v1/release_service.go b/backend/api/v1/release_service.go
index 06a68f14cd..1ea874ab2a 100644
--- a/backend/api/v1/release_service.go
+++ b/backend/api/v1/release_service.go
@@ -449,12 +449,12 @@ func convertToReleaseFiles(ctx context.Context, s *store.Store, files []*storepb
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheetUID from %q", f.Sheet)
}
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &projectID})
+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)
+ if sheet.ProjectID != projectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)
}
v1Files = append(v1Files, &v1pb.Release_File{
Id: f.Id,
@@ -491,16 +491,12 @@ func convertReleaseFiles(ctx context.Context, s *store.Store, files []*v1pb.Rele
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheetUID from %q", f.Sheet)
}
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{
- UID: &sheetUID,
- ProjectID: &projectID,
- LoadFull: false,
- })
+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)
+ if sheet.ProjectID != projectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)
}
rFiles = append(rFiles, &storepb.ReleasePayload_File{
@@ -558,16 +554,12 @@ func validateAndSanitizeReleaseFiles(ctx context.Context, s *store.Store, files
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet UID from %q", f.Sheet)
}
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{
- UID: &sheetUID,
- ProjectID: &projectID,
- LoadFull: true,
- })
+ sheet, err := s.GetSheetFull(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)
+ if sheet.ProjectID != projectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)
}
f.Statement = []byte(sheet.Statement)
f.SheetSha256 = sheet.GetSha256Hex()
diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.go
index 9672e81dc5..d1aa7fffde 100644
--- a/backend/api/v1/release_service_check.go
+++ b/backend/api/v1/release_service_check.go
@@ -446,7 +446,9 @@ func (s *ReleaseService) checkReleaseDeclarative(ctx context.Context, files []*v
// Get sheet statement from revision
_, sheetUID, err := common.GetProjectResourceIDSheetUID(revisions[0].Payload.Sheet)
if err == nil {
- previousUserSDL, _ = s.store.GetSheetStatementByID(ctx, sheetUID)
+ if sheet, err := s.store.GetSheetFull(ctx, sheetUID); err == nil {
+ previousUserSDL = sheet.Statement
+ }
}
}
diff --git a/backend/api/v1/revision_service.go b/backend/api/v1/revision_service.go
index 290aa5e8da..d39ab49645 100644
--- a/backend/api/v1/revision_service.go
+++ b/backend/api/v1/revision_service.go
@@ -131,11 +131,11 @@ func (s *RevisionService) CreateRevision(
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Wrapf(err, "failed to get sheet from %v", request.Revision.Sheet))
}
- sheet, err := s.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &sheetProjectID})
+ sheet, err := s.store.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get sheet"))
}
- if sheet == nil {
+ if sheet.ProjectID != sheetProjectID {
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID))
}
@@ -278,12 +278,12 @@ func convertToRevision(ctx context.Context, s *store.Store, parent string, revis
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheetUID from %q", revision.Payload.Sheet)
}
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &sheetProjectID})
+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %q", revision.Payload.Sheet)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID)
+ if sheet.ProjectID != sheetProjectID {
+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, sheetProjectID)
}
taskRunName := revision.Payload.TaskRun
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 88a6531431..1eacb992f7 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -913,10 +913,11 @@ func (s *RolloutService) PreviewTaskRunRollback(ctx context.Context, req *connec
if taskRun.SheetUID == nil {
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task run %v has no sheet", taskRun.ID))
}
- statements, err := s.store.GetSheetStatementByID(ctx, *taskRun.SheetUID)
+ sheet, err := s.store.GetSheetFull(ctx, *taskRun.SheetUID)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get sheet statements, error: %v", err))
}
+ statements := sheet.Statement
var results []string
for _, item := range backupDetail.Items {
diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.go
index 19dd24f07d..0dae4435ed 100644
--- a/backend/api/v1/rollout_service_converter.go
+++ b/backend/api/v1/rollout_service_converter.go
@@ -56,13 +56,10 @@ func convertToTaskRun(ctx context.Context, s *store.Store, stateCfg *state.State
}
if taskRun.SheetUID != nil && *taskRun.SheetUID != 0 {
- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: taskRun.SheetUID})
+ sheet, err := s.GetSheetMetadata(ctx, *taskRun.SheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet")
}
- if sheet == nil {
- return nil, errors.Errorf("sheet not found for uid %d", *taskRun.SheetUID)
- }
t.Sheet = common.FormatSheet(taskRun.ProjectID, sheet.UID)
}
diff --git a/backend/api/v1/sheet_service.go b/backend/api/v1/sheet_service.go
index befe8f55a7..6181461f95 100644
--- a/backend/api/v1/sheet_service.go
+++ b/backend/api/v1/sheet_service.go
@@ -155,14 +155,22 @@ func (s *SheetService) GetSheet(ctx context.Context, request *connect.Request[v1
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("project with resource id %q had deleted", projectResourceID))
}
- find := &store.FindSheetMessage{
- ProjectID: &project.ResourceID,
- UID: &sheetUID,
- LoadFull: request.Msg.Raw,
+ var sheet *store.SheetMessage
+ var sheetErr error
+ if request.Msg.Raw {
+ sheet, sheetErr = s.store.GetSheetFull(ctx, sheetUID)
+ } else {
+ sheet, sheetErr = s.store.GetSheetMetadata(ctx, sheetUID)
}
- sheet, err := s.findSheet(ctx, find)
- if err != nil {
- return nil, err
+ if sheetErr != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(sheetErr, "failed to get sheet"))
+ }
+ if sheet == nil {
+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("cannot find the sheet"))
+ }
+ // Verify project matches
+ if sheet.ProjectID != project.ResourceID {
+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sheet not found in project"))
}
v1pbSheet, err := s.convertToAPISheetMessage(ctx, sheet)
@@ -172,17 +180,6 @@ func (s *SheetService) GetSheet(ctx context.Context, request *connect.Request[v1
return connect.NewResponse(v1pbSheet), nil
}
-func (s *SheetService) findSheet(ctx context.Context, find *store.FindSheetMessage) (*store.SheetMessage, error) {
- sheet, err := s.store.GetSheet(ctx, find)
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get sheet"))
- }
- if sheet == nil {
- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("cannot find the sheet"))
- }
- return sheet, nil
-}
-
func (s *SheetService) convertToAPISheetMessage(ctx context.Context, sheet *store.SheetMessage) (*v1pb.Sheet, error) {
creator, err := s.store.GetUserByEmail(ctx, sheet.Creator)
if err != nil {
diff --git a/backend/runner/approval/runner.go b/backend/runner/approval/runner.go
index a81bdf91d0..535c2b72dd 100644
--- a/backend/runner/approval/runner.go
+++ b/backend/runner/approval/runner.go
@@ -502,11 +502,11 @@ func (r *Runner) buildCELVariablesForDatabaseChange(ctx context.Context, issue *
taskStatement := ""
sheetUID := int(task.Payload.GetSheetId())
if sheetUID != 0 {
- statement, err := r.store.GetSheetStatementByID(ctx, sheetUID)
+ sheet, err := r.store.GetSheetFull(ctx, sheetUID)
if err != nil {
return nil, true, errors.Wrapf(err, "failed to get statement in sheet %v", sheetUID)
}
- taskStatement = statement
+ taskStatement = sheet.Statement
}
var environmentID string
diff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.go
index 6ebf372e4c..2f86f4f004 100644
--- a/backend/runner/plancheck/ghost_sync_executor.go
+++ b/backend/runner/plancheck/ghost_sync_executor.go
@@ -84,17 +84,11 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRu
}
sheetUID := int(config.SheetUid)
- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})
+ sheet, err := e.store.GetSheetFull(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found", sheetUID)
- }
- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to get sheet statement %d", sheetUID)
- }
+ statement := sheet.Statement
// Database secrets feature has been removed
// To avoid leaking the rendered statement, the error message should use the original statement and not the rendered statement.
diff --git a/backend/runner/plancheck/statement_advise_executor.go b/backend/runner/plancheck/statement_advise_executor.go
index 717c2ade66..64a017ac42 100644
--- a/backend/runner/plancheck/statement_advise_executor.go
+++ b/backend/runner/plancheck/statement_advise_executor.go
@@ -46,13 +46,10 @@ type StatementAdviseExecutor struct {
// Run will run the plan check statement advise executor once, and run its sub-advisors one-by-one.
func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {
sheetUID := int(config.SheetUid)
- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})
+ sheet, err := e.store.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found", sheetUID)
- }
if sheet.Size > common.MaxSheetCheckSize {
return []*storepb.PlanCheckRunResult_Result{
{
@@ -63,10 +60,11 @@ func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanC
},
}, nil
}
- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)
+ fullSheet, err := e.store.GetSheetFull(ctx, sheetUID)
if err != nil {
return nil, err
}
+ statement := fullSheet.Statement
enablePriorBackup := config.EnablePriorBackup
enableGhost := config.EnableGhost
enableSDL := config.EnableSdl
diff --git a/backend/runner/plancheck/statement_report_executor.go b/backend/runner/plancheck/statement_report_executor.go
index 809ba79b90..73338b3afa 100644
--- a/backend/runner/plancheck/statement_report_executor.go
+++ b/backend/runner/plancheck/statement_report_executor.go
@@ -49,13 +49,10 @@ type StatementReportExecutor struct {
// Run runs the statement report executor.
func (e *StatementReportExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {
sheetUID := int(config.SheetUid)
- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})
+ sheet, err := e.store.GetSheetMetadata(ctx, sheetUID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)
}
- if sheet == nil {
- return nil, errors.Errorf("sheet %d not found", sheetUID)
- }
if sheet.Size > common.MaxSheetCheckSize {
return []*storepb.PlanCheckRunResult_Result{
{
@@ -66,10 +63,11 @@ func (e *StatementReportExecutor) Run(ctx context.Context, config *storepb.PlanC
},
}, nil
}
- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)
+ fullSheet, err := e.store.GetSheetFull(ctx, sheetUID)
if err != nil {
return nil, err
}
+ statement := fullSheet.Statement
instance, err := e.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &config.InstanceId})
if err != nil {
diff --git a/backend/runner/taskrun/data_export_executor.go b/backend/runner/taskrun/data_export_executor.go
index 5bff6628b8..f879f0549c 100644
--- a/backend/runner/taskrun/data_export_executor.go
+++ b/backend/runner/taskrun/data_export_executor.go
@@ -74,10 +74,11 @@ func (exec *DataExportExecutor) RunOnce(ctx context.Context, _ context.Context,
return true, nil, errors.Errorf("instance not found")
}
- statement, err := exec.store.GetSheetStatementByID(ctx, int(task.Payload.GetSheetId()))
+ sheet, err := exec.store.GetSheetFull(ctx, int(task.Payload.GetSheetId()))
if err != nil {
return true, nil, err
}
+ statement := sheet.Statement
dataSource := apiv1.GetQueriableDataSource(instance)
creatorUser, err := exec.store.GetUserByEmail(ctx, issue.CreatorEmail)
diff --git a/backend/runner/taskrun/database_create_executor.go b/backend/runner/taskrun/database_create_executor.go
index 69882c9926..329d8eda92 100644
--- a/backend/runner/taskrun/database_create_executor.go
+++ b/backend/runner/taskrun/database_create_executor.go
@@ -42,17 +42,11 @@ type DatabaseCreateExecutor struct {
// RunOnce will run the database create task executor once.
func (exec *DatabaseCreateExecutor) RunOnce(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, _ int) (terminated bool, result *storepb.TaskRunResult, err error) {
sheetID := int(task.Payload.GetSheetId())
- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)
- if err != nil {
- return true, nil, errors.Wrapf(err, "failed to get sheet statement of sheet: %d", sheetID)
- }
- sheet, err := exec.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetID})
+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)
if err != nil {
return true, nil, errors.Wrapf(err, "failed to get sheet: %d", sheetID)
}
- if sheet == nil {
- return true, nil, errors.Errorf("sheet not found: %d", sheetID)
- }
+ statement := sheet.Statement
statement = strings.TrimSpace(statement)
if statement == "" {
diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.go
index ecad69a3eb..550da0470d 100644
--- a/backend/runner/taskrun/database_migrate_executor.go
+++ b/backend/runner/taskrun/database_migrate_executor.go
@@ -63,10 +63,11 @@ func (exec *DatabaseMigrateExecutor) RunOnce(ctx context.Context, driverCtx cont
func (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int) (bool, *storepb.TaskRunResult, error) {
sheetID := int(task.Payload.GetSheetId())
- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)
+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)
if err != nil {
return true, nil, err
}
+ statement := sheet.Statement
// Handle prior backup if enabled.
// TransformDMLToSelect will automatically filter out DDL statements,
@@ -134,10 +135,11 @@ func (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Con
func (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int) (bool, *storepb.TaskRunResult, error) {
sheetID := int(task.Payload.GetSheetId())
- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)
+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)
if err != nil {
return true, nil, err
}
+ statement := sheet.Statement
flags := task.Payload.GetFlags()
instance, err := exec.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.go
index 26d1cebce4..b1d4a07114 100644
--- a/backend/runner/taskrun/executor.go
+++ b/backend/runner/taskrun/executor.go
@@ -199,13 +199,10 @@ func getMigrationInfo(ctx context.Context, stores *store.Store, profile *config.
}
if sheetID != nil {
- sheet, err := stores.GetSheet(ctx, &store.FindSheetMessage{UID: sheetID})
+ sheet, err := stores.GetSheetMetadata(ctx, *sheetID)
if err != nil {
return nil, errors.Wrapf(err, "failed to get sheet")
}
- if sheet == nil {
- return nil, errors.Errorf("sheet not found")
- }
mc.sheet = sheet
mc.sheetName = common.FormatSheet(pipeline.ProjectID, sheet.UID)
}
diff --git a/backend/runner/taskrun/schema_update_sdl_executor.go b/backend/runner/taskrun/schema_update_sdl_executor.go
index dae0594d46..7164bd5367 100644
--- a/backend/runner/taskrun/schema_update_sdl_executor.go
+++ b/backend/runner/taskrun/schema_update_sdl_executor.go
@@ -52,10 +52,11 @@ func (exec *SchemaDeclareExecutor) RunOnce(ctx context.Context, driverCtx contex
}
sheetID := int(task.Payload.GetSheetId())
- sheetContent, err := exec.store.GetSheetStatementByID(ctx, sheetID)
+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)
if err != nil {
return true, nil, err
}
+ sheetContent := sheet.Statement
execFunc := func(ctx context.Context, execStatement string, driver db.Driver, opts db.ExecuteOptions) error {
opts.LogComputeDiffStart()
@@ -165,10 +166,11 @@ func getPreviousSuccessfulSDLAndSchema(ctx context.Context, s *store.Store, inst
}
// Get the sheet content (original SDL text)
- previousUserSDLText, err = s.GetSheetStatementByID(ctx, sheetID)
+ sheet, err := s.GetSheetFull(ctx, sheetID)
if err != nil {
return "", nil, errors.Wrapf(err, "failed to get sheet statement for previous SDL changelog sheet ID %d", sheetID)
}
+ previousUserSDLText = sheet.Statement
}
// Get the previous schema from sync history
diff --git a/backend/store/sheet.go b/backend/store/sheet.go
index a23deee6cf..7443e56cd3 100644
--- a/backend/store/sheet.go
+++ b/backend/store/sheet.go
@@ -39,65 +39,44 @@ func (s *SheetMessage) GetSha256Hex() string {
return hex.EncodeToString(s.Sha256)
}
-// FindSheetMessage is the API message for finding sheets.
-type FindSheetMessage struct {
- ProjectID *string
- UID *int
- // LoadFull is used if we want to load the full sheet.
- LoadFull bool
-}
-
-// GetSheetStatementByID gets the statement of a sheet by ID.
-func (s *Store) GetSheetStatementByID(ctx context.Context, id int) (string, error) {
- if v, ok := s.sheetStatementCache.Get(id); ok && s.enableCache {
+// GetSheetMetadata gets a sheet with truncated statement (max 2MB).
+// Use this when you need to check sheet.Size or other metadata before processing.
+// Statement field will be truncated to MaxSheetSize (2MB).
+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error) {
+ if v, ok := s.sheetMetadataCache.Get(id); ok && s.enableCache {
return v, nil
}
- sheet, err := s.GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: true})
+ sheet, err := s.getSheet(ctx, id, false)
if err != nil {
- return "", err
- }
- if sheet == nil {
- return "", errors.Errorf("sheet not found with id %d", id)
+ return nil, err
}
- statement := sheet.Statement
- s.sheetStatementCache.Add(id, statement)
- return statement, nil
+ s.sheetMetadataCache.Add(id, sheet)
+ return sheet, nil
}
-// GetSheet gets a sheet.
-func (s *Store) GetSheet(ctx context.Context, find *FindSheetMessage) (*SheetMessage, error) {
- shouldCache := !find.LoadFull && find.UID != nil
- if shouldCache {
- if v, ok := s.sheetCache.Get(*find.UID); ok && s.enableCache {
- return v, nil
- }
+// GetSheetFull gets a sheet with the complete statement.
+// Use this when you need the full statement for execution or processing.
+// Statement field contains the complete content regardless of size.
+func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error) {
+ if v, ok := s.sheetFullCache.Get(id); ok && s.enableCache {
+ return v, nil
}
- sheets, err := s.listSheets(ctx, find)
+ sheet, err := s.getSheet(ctx, id, true)
if err != nil {
return nil, err
}
- if len(sheets) == 0 {
- return nil, nil
- }
- if len(sheets) > 1 {
- return nil, errors.Errorf("expected 1 sheet, got %d", len(sheets))
- }
- sheet := sheets[0]
-
- if shouldCache {
- s.sheetCache.Add(sheet.UID, sheet)
- }
+ s.sheetFullCache.Add(id, sheet)
return sheet, nil
}
-// listSheets returns a list of sheets.
-func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*SheetMessage, error) {
+// getSheet is the internal helper for fetching a single sheet by ID.
+func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error) {
statementField := fmt.Sprintf("LEFT(sheet_blob.content, %d)", common.MaxSheetSize)
- if find.LoadFull {
+ if loadFull {
statementField = "sheet_blob.content"
}
@@ -114,14 +93,7 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Shee
OCTET_LENGTH(sheet_blob.content)
FROM sheet
LEFT JOIN sheet_blob ON sheet.sha256 = sheet_blob.sha256
- WHERE TRUE`, statementField))
-
- if v := find.UID; v != nil {
- q.And("sheet.id = ?", *v)
- }
- if v := find.ProjectID; v != nil {
- q.And("sheet.project = ?", *v)
- }
+ WHERE sheet.id = ?`, statementField), id)
query, args, err := q.ToSQL()
if err != nil {
@@ -140,9 +112,9 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Shee
}
defer rows.Close()
- var sheets []*SheetMessage
- for rows.Next() {
- var sheet SheetMessage
+ var sheet *SheetMessage
+ if rows.Next() {
+ sheet = &SheetMessage{}
var payload []byte
if err := rows.Scan(
&sheet.UID,
@@ -162,9 +134,8 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Shee
return nil, err
}
sheet.Payload = sheetPayload
-
- sheets = append(sheets, &sheet)
}
+
if err := rows.Err(); err != nil {
return nil, err
}
@@ -172,7 +143,11 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Shee
return nil, err
}
- return sheets, nil
+ if sheet == nil {
+ return nil, errors.Errorf("sheet not found with id %d", id)
+ }
+
+ return sheet, nil
}
// CreateSheets creates new sheets.
diff --git a/backend/store/store.go b/backend/store/store.go
index 65449987b4..b57c02b005 100644
--- a/backend/store/store.go
+++ b/backend/store/store.go
@@ -30,11 +30,11 @@ type Store struct {
settingCache *lru.Cache[storepb.SettingName, *SettingMessage]
rolesCache *lru.Cache[string, *RoleMessage]
groupCache *lru.Cache[string, *GroupMessage]
- sheetCache *lru.Cache[int, *SheetMessage]
+ sheetMetadataCache *lru.Cache[int, *SheetMessage]
// Large objects.
- sheetStatementCache *lru.Cache[int, string]
- dbMetadataCache *lru.Cache[string, *model.DatabaseMetadata]
+ sheetFullCache *lru.Cache[int, *SheetMessage]
+ dbMetadataCache *lru.Cache[string, *model.DatabaseMetadata]
}
// New creates a new instance of Store.
@@ -80,11 +80,11 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {
if err != nil {
return nil, err
}
- sheetCache, err := lru.New[int, *SheetMessage](64)
+ sheetMetadataCache, err := lru.New[int, *SheetMessage](64)
if err != nil {
return nil, err
}
- sheetStatementCache, err := lru.New[int, string](10)
+ sheetFullCache, err := lru.New[int, *SheetMessage](10)
if err != nil {
return nil, err
}
@@ -118,8 +118,8 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {
pipelineCache: pipelineCache,
settingCache: settingCache,
rolesCache: rolesCache,
- sheetCache: sheetCache,
- sheetStatementCache: sheetStatementCache,
+ sheetMetadataCache: sheetMetadataCache,
+ sheetFullCache: sheetFullCache,
dbMetadataCache: dbMetadataCache,
groupCache: groupCache,
}
diff --git a/docs/plans/2025-12-16-sheet-api-redesign.md b/docs/plans/2025-12-16-sheet-api-redesign.md
new file mode 100644
index 0000000000..77a867ee82
--- /dev/null
+++ b/docs/plans/2025-12-16-sheet-api-redesign.md
@@ -0,0 +1,263 @@
+# Sheet API Redesign
+
+## Problem
+
+The current `GetSheet` and `GetSheetStatementByID` APIs are confusing for callers:
+
+1. **Unclear method choice**: Callers don't know when to use `GetSheet` vs `GetSheetStatementByID`
+2. **Confusing LoadFull flag**: The `LoadFull` boolean in `FindSheetMessage` is not self-documenting
+3. **Dual caching complexity**: Two separate caches (`sheetCache` and `sheetStatementCache`) with different behaviors make reasoning difficult
+4. **Dual-call pattern**: Many callers make two database calls when they need both metadata and full statement
+
+### Current Issues
+
+**Pattern 1: Statement-only callers** (5 locations)
+```go
+statement, err := store.GetSheetStatementByID(ctx, sheetID)
+```
+Uses `sheetStatementCache` (10 entries).
+
+**Pattern 2: Metadata + Statement callers** (4 locations)
+```go
+sheet, err := store.GetSheet(ctx, &FindSheetMessage{UID: &id})
+// Check sheet.Size > MaxSheetCheckSize
+statement, err := store.GetSheetStatementByID(ctx, id)
+```
+Makes two DB queries because the first call uses `sheetCache` (truncated) and second uses `sheetStatementCache`.
+
+**Pattern 3: API/Display callers**
+```go
+sheet, err := store.GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: raw})
+```
+LoadFull flag toggles between truncated (2MB) and full statement.
+
+## Solution
+
+Replace the confusing APIs with two purpose-specific methods that clearly communicate intent.
+
+### API Design
+
+**New Methods:**
+
+```go
+// GetSheetMetadata gets a sheet with truncated statement (max 2MB).
+// Use this when you need to check sheet.Size or other metadata before processing.
+// Statement field will be truncated to MaxSheetSize (2MB).
+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error)
+
+// GetSheetFull gets a sheet with the complete statement.
+// Use this when you need the full statement for execution or processing.
+// Statement field contains the complete content regardless of size.
+func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error)
+```
+
+**Remove:**
+```go
+// Delete these
+GetSheetStatementByID(ctx context.Context, id int) (string, error)
+GetSheet(ctx context.Context, find *FindSheetMessage) (*SheetMessage, error)
+FindSheetMessage
+```
+
+**Internal helper (private):**
+```go
+// getSheet is the internal implementation shared by both methods
+func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error)
+```
+
+### Caching Strategy
+
+**Two separate caches with renamed, clearer purpose:**
+
+```go
+type Store struct {
+ // sheetMetadataCache stores sheets with truncated statements (max 2MB)
+ // Size: 64 entries - larger since metadata checks are frequent
+ sheetMetadataCache *lru.Cache[int, *SheetMessage]
+
+ // sheetFullCache stores complete sheets with full statements
+ // Size: 10 entries - smaller since full sheets can be huge
+ sheetFullCache *lru.Cache[int, *SheetMessage]
+}
+```
+
+**Cache behavior:**
+
+- `GetSheetMetadata()`:
+ - Checks `sheetMetadataCache` first
+ - On miss: queries DB with `LEFT(sheet_blob.content, MaxSheetSize)`
+ - Stores result in `sheetMetadataCache`
+
+- `GetSheetFull()`:
+ - Checks `sheetFullCache` first
+ - On miss: queries DB with full `sheet_blob.content`
+ - Stores result in `sheetFullCache`
+
+If a caller does `GetSheetMetadata()` then `GetSheetFull()`, the second call will query the database. This is expected and acceptable - they explicitly want the full sheet.
+
+### Migration Strategy
+
+**Pattern 1: Statement-only callers** (5 callers)
+- Before: `GetSheetStatementByID(ctx, id)`
+- After: `GetSheetFull(ctx, id)` and use `.Statement` field
+- Files: data_export_executor.go, database_migrate_executor.go, approval/runner.go
+
+**Pattern 2: Metadata + Statement callers** (4 callers)
+- Before:
+ ```go
+ sheet, err := GetSheet(ctx, &FindSheetMessage{UID: &id})
+ if sheet.Size > common.MaxSheetCheckSize { return warning }
+ statement, err := GetSheetStatementByID(ctx, id)
+ ```
+- After:
+ ```go
+ sheet, err := GetSheetMetadata(ctx, id)
+ if sheet.Size > common.MaxSheetCheckSize { return warning }
+ fullSheet, err := GetSheetFull(ctx, id)
+ statement := fullSheet.Statement
+ ```
+- Files: statement_advise_executor.go, ghost_sync_executor.go, statement_report_executor.go
+
+**Pattern 3: API/Display callers**
+- Before: `GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: raw})`
+- After:
+ ```go
+ if raw {
+ sheet, err := GetSheetFull(ctx, id)
+ } else {
+ sheet, err := GetSheetMetadata(ctx, id)
+ }
+ ```
+- Files: sheet_service.go, release_service.go
+
+**Pattern 4: Mixed callers** (database_create_executor.go)
+- Currently calls both `GetSheetStatementByID` and `GetSheet` separately
+- After: `GetSheetFull(ctx, id)` - single call gets everything
+
+### Implementation
+
+```go
+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error) {
+ if v, ok := s.sheetMetadataCache.Get(id); ok && s.enableCache {
+ return v, nil
+ }
+
+ sheet, err := s.getSheet(ctx, id, false)
+ if err != nil {
+ return nil, err
+ }
+
+ s.sheetMetadataCache.Add(id, sheet)
+ return sheet, nil
+}
+
+func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error) {
+ if v, ok := s.sheetFullCache.Get(id); ok && s.enableCache {
+ return v, nil
+ }
+
+ sheet, err := s.getSheet(ctx, id, true)
+ if err != nil {
+ return nil, err
+ }
+
+ s.sheetFullCache.Add(id, sheet)
+ return sheet, nil
+}
+
+func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error) {
+ statementField := fmt.Sprintf("LEFT(sheet_blob.content, %d)", common.MaxSheetSize)
+ if loadFull {
+ statementField = "sheet_blob.content"
+ }
+
+ q := qb.Q().Space(fmt.Sprintf(`
+ SELECT
+ sheet.id,
+ sheet.creator,
+ sheet.created_at,
+ sheet.project,
+ sheet.name,
+ %s,
+ sheet.sha256,
+ sheet.payload,
+ OCTET_LENGTH(sheet_blob.content)
+ FROM sheet
+ LEFT JOIN sheet_blob ON sheet.sha256 = sheet_blob.sha256
+ WHERE sheet.id = ?`, statementField), id)
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+
+ tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+ if err != nil {
+ return nil, err
+ }
+ defer tx.Rollback()
+
+ rows, err := tx.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ var sheet *SheetMessage
+ if rows.Next() {
+ sheet = &SheetMessage{}
+ var payload []byte
+ if err := rows.Scan(
+ &sheet.UID,
+ &sheet.Creator,
+ &sheet.CreatedAt,
+ &sheet.ProjectID,
+ &sheet.Title,
+ &sheet.Statement,
+ &sheet.Sha256,
+ &payload,
+ &sheet.Size,
+ ); err != nil {
+ return nil, err
+ }
+ sheetPayload := &storepb.SheetPayload{}
+ if err := common.ProtojsonUnmarshaler.Unmarshal(payload, sheetPayload); err != nil {
+ return nil, err
+ }
+ sheet.Payload = sheetPayload
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ if err := tx.Commit(); err != nil {
+ return nil, err
+ }
+
+ if sheet == nil {
+ return nil, errors.Errorf("sheet not found with id %d", id)
+ }
+
+ return sheet, nil
+}
+```
+
+**Edge cases handled:**
+- Sheet not found: return error with clear message
+- Cache disabled: still works, queries DB every time
+- Multiple sheets returned: impossible with ID-based WHERE clause
+
+## Benefits
+
+1. **Clear intent**: Method names communicate exactly what you get
+2. **No confusing flags**: No LoadFull boolean to explain
+3. **Simpler mental model**: Two caches with obvious purposes
+4. **Consistent behavior**: Each method has one caching strategy
+5. **Better errors**: Single method to fix when sheet not found
+6. **Type safety**: Return full SheetMessage instead of string-only
+
+## Trade-offs
+
+- Pattern 2 callers still make two calls (metadata check + full fetch), but now it's explicit and intentional
+- Migration effort across ~15 call sites
+- Two methods instead of one flexible method (but flexibility was the source of confusion)