Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: redesign Sheet API with GetSheetMetadata/GetSheetFull (#18499)
Danny Xu3675a417bf40
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.goindex db46d9f9bb..d9e1c9512d 100644--- a/backend/api/v1/plan_service.go+++ b/backend/api/v1/plan_service.go@@ -609,14 +609,11 @@ func (s *PlanService) UpdatePlan(ctx context.Context, request *connect.Request[vreturn nil}- sheet, err := s.store.GetSheet(ctx, &store.FindSheetMessage{- UID: &sheetUID,- ProjectID: &sheetProjectID,- })+ sheet, err := s.store.GetSheetMetadata(ctx, sheetUID)if err != nil {return connect.NewError(connect.CodeInternal, errors.Errorf("failed to get sheet %q: %v", oldSheetName, err))}- if sheet == nil {+ if sheet.ProjectID != sheetProjectID {return connect.NewError(connect.CodeNotFound, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID))}doUpdate = truediff --git a/backend/api/v1/plan_service_converter.go b/backend/api/v1/plan_service_converter.goindex 825cd4d392..f0d1b1518e 100644--- a/backend/api/v1/plan_service_converter.go+++ b/backend/api/v1/plan_service_converter.go@@ -298,12 +298,12 @@ func convertToPlanCheckRun(ctx context.Context, s *store.Store, projectID string}if sheetUID := int(run.Config.GetSheetUid()); sheetUID != 0 {- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &projectID})+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet")}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)+ if sheet.ProjectID != projectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)}converted.Sheet = common.FormatSheet(projectID, sheet.UID)}diff --git a/backend/api/v1/plan_service_plan_check.go b/backend/api/v1/plan_service_plan_check.goindex a48dfe6eeb..28b07bacbb 100644--- a/backend/api/v1/plan_service_plan_check.go+++ b/backend/api/v1/plan_service_plan_check.go@@ -178,15 +178,12 @@ func getPlanCheckRunsFromChangeDatabaseConfigForDatabase(ctx context.Context, sif instance == nil {return nil, errors.Errorf("instance %q not found", database.InstanceID)}- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{- UID: &sheetUID,- ProjectID: &plan.ProjectID,- })+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, plan.ProjectID)+ if sheet.ProjectID != plan.ProjectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, plan.ProjectID)}enableSDL := config.Type == storepb.PlanConfig_ChangeDatabaseConfig_SDLdiff --git a/backend/api/v1/release_service.go b/backend/api/v1/release_service.goindex 06a68f14cd..1ea874ab2a 100644--- a/backend/api/v1/release_service.go+++ b/backend/api/v1/release_service.go@@ -449,12 +449,12 @@ func convertToReleaseFiles(ctx context.Context, s *store.Store, files []*storepbif err != nil {return nil, errors.Wrapf(err, "failed to get sheetUID from %q", f.Sheet)}- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &projectID})+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)+ if sheet.ProjectID != projectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)}v1Files = append(v1Files, &v1pb.Release_File{Id: f.Id,@@ -491,16 +491,12 @@ func convertReleaseFiles(ctx context.Context, s *store.Store, files []*v1pb.Releif err != nil {return nil, errors.Wrapf(err, "failed to get sheetUID from %q", f.Sheet)}- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{- UID: &sheetUID,- ProjectID: &projectID,- LoadFull: false,- })+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)+ if sheet.ProjectID != projectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)}rFiles = append(rFiles, &storepb.ReleasePayload_File{@@ -558,16 +554,12 @@ func validateAndSanitizeReleaseFiles(ctx context.Context, s *store.Store, filesif err != nil {return nil, errors.Wrapf(err, "failed to get sheet UID from %q", f.Sheet)}- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{- UID: &sheetUID,- ProjectID: &projectID,- LoadFull: true,- })+ sheet, err := s.GetSheetFull(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %q", f.Sheet)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, projectID)+ if sheet.ProjectID != projectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, projectID)}f.Statement = []byte(sheet.Statement)f.SheetSha256 = sheet.GetSha256Hex()diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.goindex 9672e81dc5..d1aa7fffde 100644--- a/backend/api/v1/release_service_check.go+++ b/backend/api/v1/release_service_check.go@@ -446,7 +446,9 @@ func (s *ReleaseService) checkReleaseDeclarative(ctx context.Context, files []*v// Get sheet statement from revision_, sheetUID, err := common.GetProjectResourceIDSheetUID(revisions[0].Payload.Sheet)if err == nil {- previousUserSDL, _ = s.store.GetSheetStatementByID(ctx, sheetUID)+ if sheet, err := s.store.GetSheetFull(ctx, sheetUID); err == nil {+ previousUserSDL = sheet.Statement+ }}}diff --git a/backend/api/v1/revision_service.go b/backend/api/v1/revision_service.goindex 290aa5e8da..d39ab49645 100644--- a/backend/api/v1/revision_service.go+++ b/backend/api/v1/revision_service.go@@ -131,11 +131,11 @@ func (s *RevisionService) CreateRevision(if err != nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Wrapf(err, "failed to get sheet from %v", request.Revision.Sheet))}- sheet, err := s.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &sheetProjectID})+ sheet, err := s.store.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get sheet"))}- if sheet == nil {+ if sheet.ProjectID != sheetProjectID {return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID))}@@ -278,12 +278,12 @@ func convertToRevision(ctx context.Context, s *store.Store, parent string, revisif err != nil {return nil, errors.Wrapf(err, "failed to get sheetUID from %q", revision.Payload.Sheet)}- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID, ProjectID: &sheetProjectID})+ sheet, err := s.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %q", revision.Payload.Sheet)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found in project %s", sheetUID, sheetProjectID)+ if sheet.ProjectID != sheetProjectID {+ return nil, errors.Errorf("sheet %d not in project %s", sheetUID, sheetProjectID)}taskRunName := revision.Payload.TaskRundiff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 88a6531431..1eacb992f7 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -913,10 +913,11 @@ func (s *RolloutService) PreviewTaskRunRollback(ctx context.Context, req *connecif taskRun.SheetUID == nil {return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("task run %v has no sheet", taskRun.ID))}- statements, err := s.store.GetSheetStatementByID(ctx, *taskRun.SheetUID)+ sheet, err := s.store.GetSheetFull(ctx, *taskRun.SheetUID)if err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to get sheet statements, error: %v", err))}+ statements := sheet.Statementvar results []stringfor _, item := range backupDetail.Items {diff --git a/backend/api/v1/rollout_service_converter.go b/backend/api/v1/rollout_service_converter.goindex 19dd24f07d..0dae4435ed 100644--- a/backend/api/v1/rollout_service_converter.go+++ b/backend/api/v1/rollout_service_converter.go@@ -56,13 +56,10 @@ func convertToTaskRun(ctx context.Context, s *store.Store, stateCfg *state.State}if taskRun.SheetUID != nil && *taskRun.SheetUID != 0 {- sheet, err := s.GetSheet(ctx, &store.FindSheetMessage{UID: taskRun.SheetUID})+ sheet, err := s.GetSheetMetadata(ctx, *taskRun.SheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet")}- if sheet == nil {- return nil, errors.Errorf("sheet not found for uid %d", *taskRun.SheetUID)- }t.Sheet = common.FormatSheet(taskRun.ProjectID, sheet.UID)}diff --git a/backend/api/v1/sheet_service.go b/backend/api/v1/sheet_service.goindex befe8f55a7..6181461f95 100644--- a/backend/api/v1/sheet_service.go+++ b/backend/api/v1/sheet_service.go@@ -155,14 +155,22 @@ func (s *SheetService) GetSheet(ctx context.Context, request *connect.Request[v1return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("project with resource id %q had deleted", projectResourceID))}- find := &store.FindSheetMessage{- ProjectID: &project.ResourceID,- UID: &sheetUID,- LoadFull: request.Msg.Raw,+ var sheet *store.SheetMessage+ var sheetErr error+ if request.Msg.Raw {+ sheet, sheetErr = s.store.GetSheetFull(ctx, sheetUID)+ } else {+ sheet, sheetErr = s.store.GetSheetMetadata(ctx, sheetUID)}- sheet, err := s.findSheet(ctx, find)- if err != nil {- return nil, err+ if sheetErr != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(sheetErr, "failed to get sheet"))+ }+ if sheet == nil {+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("cannot find the sheet"))+ }+ // Verify project matches+ if sheet.ProjectID != project.ResourceID {+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("sheet not found in project"))}v1pbSheet, err := s.convertToAPISheetMessage(ctx, sheet)@@ -172,17 +180,6 @@ func (s *SheetService) GetSheet(ctx context.Context, request *connect.Request[v1return connect.NewResponse(v1pbSheet), nil}-func (s *SheetService) findSheet(ctx context.Context, find *store.FindSheetMessage) (*store.SheetMessage, error) {- sheet, err := s.store.GetSheet(ctx, find)- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get sheet"))- }- if sheet == nil {- return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("cannot find the sheet"))- }- return sheet, nil-}-func (s *SheetService) convertToAPISheetMessage(ctx context.Context, sheet *store.SheetMessage) (*v1pb.Sheet, error) {creator, err := s.store.GetUserByEmail(ctx, sheet.Creator)if err != nil {diff --git a/backend/runner/approval/runner.go b/backend/runner/approval/runner.goindex a81bdf91d0..535c2b72dd 100644--- a/backend/runner/approval/runner.go+++ b/backend/runner/approval/runner.go@@ -502,11 +502,11 @@ func (r *Runner) buildCELVariablesForDatabaseChange(ctx context.Context, issue *taskStatement := ""sheetUID := int(task.Payload.GetSheetId())if sheetUID != 0 {- statement, err := r.store.GetSheetStatementByID(ctx, sheetUID)+ sheet, err := r.store.GetSheetFull(ctx, sheetUID)if err != nil {return nil, true, errors.Wrapf(err, "failed to get statement in sheet %v", sheetUID)}- taskStatement = statement+ taskStatement = sheet.Statement}var environmentID stringdiff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.goindex 6ebf372e4c..2f86f4f004 100644--- a/backend/runner/plancheck/ghost_sync_executor.go+++ b/backend/runner/plancheck/ghost_sync_executor.go@@ -84,17 +84,11 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRu}sheetUID := int(config.SheetUid)- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})+ sheet, err := e.store.GetSheetFull(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found", sheetUID)- }- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)- if err != nil {- return nil, errors.Wrapf(err, "failed to get sheet statement %d", sheetUID)- }+ statement := sheet.Statement// Database secrets feature has been removed// To avoid leaking the rendered statement, the error message should use the original statement and not the rendered statement.diff --git a/backend/runner/plancheck/statement_advise_executor.go b/backend/runner/plancheck/statement_advise_executor.goindex 717c2ade66..64a017ac42 100644--- a/backend/runner/plancheck/statement_advise_executor.go+++ b/backend/runner/plancheck/statement_advise_executor.go@@ -46,13 +46,10 @@ type StatementAdviseExecutor struct {// Run will run the plan check statement advise executor once, and run its sub-advisors one-by-one.func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {sheetUID := int(config.SheetUid)- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})+ sheet, err := e.store.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found", sheetUID)- }if sheet.Size > common.MaxSheetCheckSize {return []*storepb.PlanCheckRunResult_Result{{@@ -63,10 +60,11 @@ func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanC},}, nil}- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)+ fullSheet, err := e.store.GetSheetFull(ctx, sheetUID)if err != nil {return nil, err}+ statement := fullSheet.StatementenablePriorBackup := config.EnablePriorBackupenableGhost := config.EnableGhostenableSDL := config.EnableSdldiff --git a/backend/runner/plancheck/statement_report_executor.go b/backend/runner/plancheck/statement_report_executor.goindex 809ba79b90..73338b3afa 100644--- a/backend/runner/plancheck/statement_report_executor.go+++ b/backend/runner/plancheck/statement_report_executor.go@@ -49,13 +49,10 @@ type StatementReportExecutor struct {// Run runs the statement report executor.func (e *StatementReportExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {sheetUID := int(config.SheetUid)- sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})+ sheet, err := e.store.GetSheetMetadata(ctx, sheetUID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet %d", sheetUID)}- if sheet == nil {- return nil, errors.Errorf("sheet %d not found", sheetUID)- }if sheet.Size > common.MaxSheetCheckSize {return []*storepb.PlanCheckRunResult_Result{{@@ -66,10 +63,11 @@ func (e *StatementReportExecutor) Run(ctx context.Context, config *storepb.PlanC},}, nil}- statement, err := e.store.GetSheetStatementByID(ctx, sheetUID)+ fullSheet, err := e.store.GetSheetFull(ctx, sheetUID)if err != nil {return nil, err}+ statement := fullSheet.Statementinstance, err := e.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &config.InstanceId})if err != nil {diff --git a/backend/runner/taskrun/data_export_executor.go b/backend/runner/taskrun/data_export_executor.goindex 5bff6628b8..f879f0549c 100644--- a/backend/runner/taskrun/data_export_executor.go+++ b/backend/runner/taskrun/data_export_executor.go@@ -74,10 +74,11 @@ func (exec *DataExportExecutor) RunOnce(ctx context.Context, _ context.Context,return true, nil, errors.Errorf("instance not found")}- statement, err := exec.store.GetSheetStatementByID(ctx, int(task.Payload.GetSheetId()))+ sheet, err := exec.store.GetSheetFull(ctx, int(task.Payload.GetSheetId()))if err != nil {return true, nil, err}+ statement := sheet.StatementdataSource := apiv1.GetQueriableDataSource(instance)creatorUser, err := exec.store.GetUserByEmail(ctx, issue.CreatorEmail)diff --git a/backend/runner/taskrun/database_create_executor.go b/backend/runner/taskrun/database_create_executor.goindex 69882c9926..329d8eda92 100644--- a/backend/runner/taskrun/database_create_executor.go+++ b/backend/runner/taskrun/database_create_executor.go@@ -42,17 +42,11 @@ type DatabaseCreateExecutor struct {// RunOnce will run the database create task executor once.func (exec *DatabaseCreateExecutor) RunOnce(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, _ int) (terminated bool, result *storepb.TaskRunResult, err error) {sheetID := int(task.Payload.GetSheetId())- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)- if err != nil {- return true, nil, errors.Wrapf(err, "failed to get sheet statement of sheet: %d", sheetID)- }- sheet, err := exec.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetID})+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)if err != nil {return true, nil, errors.Wrapf(err, "failed to get sheet: %d", sheetID)}- if sheet == nil {- return true, nil, errors.Errorf("sheet not found: %d", sheetID)- }+ statement := sheet.Statementstatement = strings.TrimSpace(statement)if statement == "" {diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.goindex ecad69a3eb..550da0470d 100644--- a/backend/runner/taskrun/database_migrate_executor.go+++ b/backend/runner/taskrun/database_migrate_executor.go@@ -63,10 +63,11 @@ func (exec *DatabaseMigrateExecutor) RunOnce(ctx context.Context, driverCtx contfunc (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int) (bool, *storepb.TaskRunResult, error) {sheetID := int(task.Payload.GetSheetId())- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)if err != nil {return true, nil, err}+ statement := sheet.Statement// Handle prior backup if enabled.// TransformDMLToSelect will automatically filter out DDL statements,@@ -134,10 +135,11 @@ func (exec *DatabaseMigrateExecutor) runMigrationWithPriorBackup(ctx context.Confunc (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int) (bool, *storepb.TaskRunResult, error) {sheetID := int(task.Payload.GetSheetId())- statement, err := exec.store.GetSheetStatementByID(ctx, sheetID)+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)if err != nil {return true, nil, err}+ statement := sheet.Statementflags := task.Payload.GetFlags()instance, err := exec.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.goindex 26d1cebce4..b1d4a07114 100644--- a/backend/runner/taskrun/executor.go+++ b/backend/runner/taskrun/executor.go@@ -199,13 +199,10 @@ func getMigrationInfo(ctx context.Context, stores *store.Store, profile *config.}if sheetID != nil {- sheet, err := stores.GetSheet(ctx, &store.FindSheetMessage{UID: sheetID})+ sheet, err := stores.GetSheetMetadata(ctx, *sheetID)if err != nil {return nil, errors.Wrapf(err, "failed to get sheet")}- if sheet == nil {- return nil, errors.Errorf("sheet not found")- }mc.sheet = sheetmc.sheetName = common.FormatSheet(pipeline.ProjectID, sheet.UID)}diff --git a/backend/runner/taskrun/schema_update_sdl_executor.go b/backend/runner/taskrun/schema_update_sdl_executor.goindex dae0594d46..7164bd5367 100644--- a/backend/runner/taskrun/schema_update_sdl_executor.go+++ b/backend/runner/taskrun/schema_update_sdl_executor.go@@ -52,10 +52,11 @@ func (exec *SchemaDeclareExecutor) RunOnce(ctx context.Context, driverCtx contex}sheetID := int(task.Payload.GetSheetId())- sheetContent, err := exec.store.GetSheetStatementByID(ctx, sheetID)+ sheet, err := exec.store.GetSheetFull(ctx, sheetID)if err != nil {return true, nil, err}+ sheetContent := sheet.StatementexecFunc := func(ctx context.Context, execStatement string, driver db.Driver, opts db.ExecuteOptions) error {opts.LogComputeDiffStart()@@ -165,10 +166,11 @@ func getPreviousSuccessfulSDLAndSchema(ctx context.Context, s *store.Store, inst}// Get the sheet content (original SDL text)- previousUserSDLText, err = s.GetSheetStatementByID(ctx, sheetID)+ sheet, err := s.GetSheetFull(ctx, sheetID)if err != nil {return "", nil, errors.Wrapf(err, "failed to get sheet statement for previous SDL changelog sheet ID %d", sheetID)}+ previousUserSDLText = sheet.Statement}// Get the previous schema from sync historydiff --git a/backend/store/sheet.go b/backend/store/sheet.goindex a23deee6cf..7443e56cd3 100644--- a/backend/store/sheet.go+++ b/backend/store/sheet.go@@ -39,65 +39,44 @@ func (s *SheetMessage) GetSha256Hex() string {return hex.EncodeToString(s.Sha256)}-// FindSheetMessage is the API message for finding sheets.-type FindSheetMessage struct {- ProjectID *string- UID *int- // LoadFull is used if we want to load the full sheet.- LoadFull bool-}--// GetSheetStatementByID gets the statement of a sheet by ID.-func (s *Store) GetSheetStatementByID(ctx context.Context, id int) (string, error) {- if v, ok := s.sheetStatementCache.Get(id); ok && s.enableCache {+// GetSheetMetadata gets a sheet with truncated statement (max 2MB).+// Use this when you need to check sheet.Size or other metadata before processing.+// Statement field will be truncated to MaxSheetSize (2MB).+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error) {+ if v, ok := s.sheetMetadataCache.Get(id); ok && s.enableCache {return v, nil}- sheet, err := s.GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: true})+ sheet, err := s.getSheet(ctx, id, false)if err != nil {- return "", err- }- if sheet == nil {- return "", errors.Errorf("sheet not found with id %d", id)+ return nil, err}- statement := sheet.Statement- s.sheetStatementCache.Add(id, statement)- return statement, nil+ s.sheetMetadataCache.Add(id, sheet)+ return sheet, nil}-// GetSheet gets a sheet.-func (s *Store) GetSheet(ctx context.Context, find *FindSheetMessage) (*SheetMessage, error) {- shouldCache := !find.LoadFull && find.UID != nil- if shouldCache {- if v, ok := s.sheetCache.Get(*find.UID); ok && s.enableCache {- return v, nil- }+// GetSheetFull gets a sheet with the complete statement.+// Use this when you need the full statement for execution or processing.+// Statement field contains the complete content regardless of size.+func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error) {+ if v, ok := s.sheetFullCache.Get(id); ok && s.enableCache {+ return v, nil}- sheets, err := s.listSheets(ctx, find)+ sheet, err := s.getSheet(ctx, id, true)if err != nil {return nil, err}- if len(sheets) == 0 {- return nil, nil- }- if len(sheets) > 1 {- return nil, errors.Errorf("expected 1 sheet, got %d", len(sheets))- }- sheet := sheets[0]-- if shouldCache {- s.sheetCache.Add(sheet.UID, sheet)- }+ s.sheetFullCache.Add(id, sheet)return sheet, nil}-// listSheets returns a list of sheets.-func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*SheetMessage, error) {+// getSheet is the internal helper for fetching a single sheet by ID.+func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error) {statementField := fmt.Sprintf("LEFT(sheet_blob.content, %d)", common.MaxSheetSize)- if find.LoadFull {+ if loadFull {statementField = "sheet_blob.content"}@@ -114,14 +93,7 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*SheeOCTET_LENGTH(sheet_blob.content)FROM sheetLEFT JOIN sheet_blob ON sheet.sha256 = sheet_blob.sha256- WHERE TRUE`, statementField))-- if v := find.UID; v != nil {- q.And("sheet.id = ?", *v)- }- if v := find.ProjectID; v != nil {- q.And("sheet.project = ?", *v)- }+ WHERE sheet.id = ?`, statementField), id)query, args, err := q.ToSQL()if err != nil {@@ -140,9 +112,9 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Shee}defer rows.Close()- var sheets []*SheetMessage- for rows.Next() {- var sheet SheetMessage+ var sheet *SheetMessage+ if rows.Next() {+ sheet = &SheetMessage{}var payload []byteif err := rows.Scan(&sheet.UID,@@ -162,9 +134,8 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Sheereturn nil, err}sheet.Payload = sheetPayload-- sheets = append(sheets, &sheet)}+if err := rows.Err(); err != nil {return nil, err}@@ -172,7 +143,11 @@ func (s *Store) listSheets(ctx context.Context, find *FindSheetMessage) ([]*Sheereturn nil, err}- return sheets, nil+ if sheet == nil {+ return nil, errors.Errorf("sheet not found with id %d", id)+ }++ return sheet, nil}// CreateSheets creates new sheets.diff --git a/backend/store/store.go b/backend/store/store.goindex 65449987b4..b57c02b005 100644--- a/backend/store/store.go+++ b/backend/store/store.go@@ -30,11 +30,11 @@ type Store struct {settingCache *lru.Cache[storepb.SettingName, *SettingMessage]rolesCache *lru.Cache[string, *RoleMessage]groupCache *lru.Cache[string, *GroupMessage]- sheetCache *lru.Cache[int, *SheetMessage]+ sheetMetadataCache *lru.Cache[int, *SheetMessage]// Large objects.- sheetStatementCache *lru.Cache[int, string]- dbMetadataCache *lru.Cache[string, *model.DatabaseMetadata]+ sheetFullCache *lru.Cache[int, *SheetMessage]+ dbMetadataCache *lru.Cache[string, *model.DatabaseMetadata]}// New creates a new instance of Store.@@ -80,11 +80,11 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {if err != nil {return nil, err}- sheetCache, err := lru.New[int, *SheetMessage](64)+ sheetMetadataCache, err := lru.New[int, *SheetMessage](64)if err != nil {return nil, err}- sheetStatementCache, err := lru.New[int, string](10)+ sheetFullCache, err := lru.New[int, *SheetMessage](10)if err != nil {return nil, err}@@ -118,8 +118,8 @@ func New(ctx context.Context, pgURL string, enableCache bool) (*Store, error) {pipelineCache: pipelineCache,settingCache: settingCache,rolesCache: rolesCache,- sheetCache: sheetCache,- sheetStatementCache: sheetStatementCache,+ sheetMetadataCache: sheetMetadataCache,+ sheetFullCache: sheetFullCache,dbMetadataCache: dbMetadataCache,groupCache: groupCache,}diff --git a/docs/plans/2025-12-16-sheet-api-redesign.md b/docs/plans/2025-12-16-sheet-api-redesign.mdnew file mode 100644index 0000000000..77a867ee82--- /dev/null+++ b/docs/plans/2025-12-16-sheet-api-redesign.md@@ -0,0 +1,263 @@+# Sheet API Redesign++## Problem++The current `GetSheet` and `GetSheetStatementByID` APIs are confusing for callers:++1. **Unclear method choice**: Callers don't know when to use `GetSheet` vs `GetSheetStatementByID`+2. **Confusing LoadFull flag**: The `LoadFull` boolean in `FindSheetMessage` is not self-documenting+3. **Dual caching complexity**: Two separate caches (`sheetCache` and `sheetStatementCache`) with different behaviors make reasoning difficult+4. **Dual-call pattern**: Many callers make two database calls when they need both metadata and full statement++### Current Issues++**Pattern 1: Statement-only callers** (5 locations)+```go+statement, err := store.GetSheetStatementByID(ctx, sheetID)+```+Uses `sheetStatementCache` (10 entries).++**Pattern 2: Metadata + Statement callers** (4 locations)+```go+sheet, err := store.GetSheet(ctx, &FindSheetMessage{UID: &id})+// Check sheet.Size > MaxSheetCheckSize+statement, err := store.GetSheetStatementByID(ctx, id)+```+Makes two DB queries because the first call uses `sheetCache` (truncated) and second uses `sheetStatementCache`.++**Pattern 3: API/Display callers**+```go+sheet, err := store.GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: raw})+```+LoadFull flag toggles between truncated (2MB) and full statement.++## Solution++Replace the confusing APIs with two purpose-specific methods that clearly communicate intent.++### API Design++**New Methods:**++```go+// GetSheetMetadata gets a sheet with truncated statement (max 2MB).+// Use this when you need to check sheet.Size or other metadata before processing.+// Statement field will be truncated to MaxSheetSize (2MB).+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error)++// GetSheetFull gets a sheet with the complete statement.+// Use this when you need the full statement for execution or processing.+// Statement field contains the complete content regardless of size.+func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error)+```++**Remove:**+```go+// Delete these+GetSheetStatementByID(ctx context.Context, id int) (string, error)+GetSheet(ctx context.Context, find *FindSheetMessage) (*SheetMessage, error)+FindSheetMessage+```++**Internal helper (private):**+```go+// getSheet is the internal implementation shared by both methods+func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error)+```++### Caching Strategy++**Two separate caches with renamed, clearer purpose:**++```go+type Store struct {+ // sheetMetadataCache stores sheets with truncated statements (max 2MB)+ // Size: 64 entries - larger since metadata checks are frequent+ sheetMetadataCache *lru.Cache[int, *SheetMessage]++ // sheetFullCache stores complete sheets with full statements+ // Size: 10 entries - smaller since full sheets can be huge+ sheetFullCache *lru.Cache[int, *SheetMessage]+}+```++**Cache behavior:**++- `GetSheetMetadata()`:+ - Checks `sheetMetadataCache` first+ - On miss: queries DB with `LEFT(sheet_blob.content, MaxSheetSize)`+ - Stores result in `sheetMetadataCache`++- `GetSheetFull()`:+ - Checks `sheetFullCache` first+ - On miss: queries DB with full `sheet_blob.content`+ - Stores result in `sheetFullCache`++If a caller does `GetSheetMetadata()` then `GetSheetFull()`, the second call will query the database. This is expected and acceptable - they explicitly want the full sheet.++### Migration Strategy++**Pattern 1: Statement-only callers** (5 callers)+- Before: `GetSheetStatementByID(ctx, id)`+- After: `GetSheetFull(ctx, id)` and use `.Statement` field+- Files: data_export_executor.go, database_migrate_executor.go, approval/runner.go++**Pattern 2: Metadata + Statement callers** (4 callers)+- Before:+ ```go+ sheet, err := GetSheet(ctx, &FindSheetMessage{UID: &id})+ if sheet.Size > common.MaxSheetCheckSize { return warning }+ statement, err := GetSheetStatementByID(ctx, id)+ ```+- After:+ ```go+ sheet, err := GetSheetMetadata(ctx, id)+ if sheet.Size > common.MaxSheetCheckSize { return warning }+ fullSheet, err := GetSheetFull(ctx, id)+ statement := fullSheet.Statement+ ```+- Files: statement_advise_executor.go, ghost_sync_executor.go, statement_report_executor.go++**Pattern 3: API/Display callers**+- Before: `GetSheet(ctx, &FindSheetMessage{UID: &id, LoadFull: raw})`+- After:+ ```go+ if raw {+ sheet, err := GetSheetFull(ctx, id)+ } else {+ sheet, err := GetSheetMetadata(ctx, id)+ }+ ```+- Files: sheet_service.go, release_service.go++**Pattern 4: Mixed callers** (database_create_executor.go)+- Currently calls both `GetSheetStatementByID` and `GetSheet` separately+- After: `GetSheetFull(ctx, id)` - single call gets everything++### Implementation++```go+func (s *Store) GetSheetMetadata(ctx context.Context, id int) (*SheetMessage, error) {+ if v, ok := s.sheetMetadataCache.Get(id); ok && s.enableCache {+ return v, nil+ }++ sheet, err := s.getSheet(ctx, id, false)+ if err != nil {+ return nil, err+ }++ s.sheetMetadataCache.Add(id, sheet)+ return sheet, nil+}++func (s *Store) GetSheetFull(ctx context.Context, id int) (*SheetMessage, error) {+ if v, ok := s.sheetFullCache.Get(id); ok && s.enableCache {+ return v, nil+ }++ sheet, err := s.getSheet(ctx, id, true)+ if err != nil {+ return nil, err+ }++ s.sheetFullCache.Add(id, sheet)+ return sheet, nil+}++func (s *Store) getSheet(ctx context.Context, id int, loadFull bool) (*SheetMessage, error) {+ statementField := fmt.Sprintf("LEFT(sheet_blob.content, %d)", common.MaxSheetSize)+ if loadFull {+ statementField = "sheet_blob.content"+ }++ q := qb.Q().Space(fmt.Sprintf(`+ SELECT+ sheet.id,+ sheet.creator,+ sheet.created_at,+ sheet.project,+ sheet.name,+ %s,+ sheet.sha256,+ sheet.payload,+ OCTET_LENGTH(sheet_blob.content)+ FROM sheet+ LEFT JOIN sheet_blob ON sheet.sha256 = sheet_blob.sha256+ WHERE sheet.id = ?`, statementField), id)++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})+ if err != nil {+ return nil, err+ }+ defer tx.Rollback()++ rows, err := tx.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ var sheet *SheetMessage+ if rows.Next() {+ sheet = &SheetMessage{}+ var payload []byte+ if err := rows.Scan(+ &sheet.UID,+ &sheet.Creator,+ &sheet.CreatedAt,+ &sheet.ProjectID,+ &sheet.Title,+ &sheet.Statement,+ &sheet.Sha256,+ &payload,+ &sheet.Size,+ ); err != nil {+ return nil, err+ }+ sheetPayload := &storepb.SheetPayload{}+ if err := common.ProtojsonUnmarshaler.Unmarshal(payload, sheetPayload); err != nil {+ return nil, err+ }+ sheet.Payload = sheetPayload+ }++ if err := rows.Err(); err != nil {+ return nil, err+ }+ if err := tx.Commit(); err != nil {+ return nil, err+ }++ if sheet == nil {+ return nil, errors.Errorf("sheet not found with id %d", id)+ }++ return sheet, nil+}+```++**Edge cases handled:**+- Sheet not found: return error with clear message+- Cache disabled: still works, queries DB every time+- Multiple sheets returned: impossible with ID-based WHERE clause++## Benefits++1. **Clear intent**: Method names communicate exactly what you get+2. **No confusing flags**: No LoadFull boolean to explain+3. **Simpler mental model**: Two caches with obvious purposes+4. **Consistent behavior**: Each method has one caching strategy+5. **Better errors**: Single method to fix when sheet not found+6. **Type safety**: Return full SheetMessage instead of string-only++## Trade-offs++- Pattern 2 callers still make two calls (metadata check + full fetch), but now it's explicit and intentional+- Migration effort across ~15 call sites+- Two methods instead of one flexible method (but flexibility was the source of confusion)