Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
perf(export): mask exports once and cut SharePoint export round trips (#21868)
p0ny2f1c7be0ce94
diff --git a/backend/api/v1/project_service_export_setting_test.go b/backend/api/v1/project_service_export_setting_test.goindex 4a4ca45177..cdde45a394 100644--- a/backend/api/v1/project_service_export_setting_test.go+++ b/backend/api/v1/project_service_export_setting_test.go@@ -29,6 +29,10 @@ func (c *fakeSharePointResolver) Resolve(_ context.Context, canonicalURL string)return c.location, c.err}+func (*fakeSharePointResolver) ResolveRecent(context.Context, string) (*sharepoint.Location, error) {+ return nil, errors.New("not used")+}+func (*fakeSharePointResolver) Upload(context.Context, *sharepoint.Location, string, []sharepoint.File) (*sharepoint.Folder, error) {return nil, errors.New("not used")}diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex 95a239ec00..592135ee45 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -171,11 +171,8 @@ func (s *SQLService) AdminExecute(ctx context.Context, stream *connect.BidiStrea}// buildExportQueryContext assembles the db.QueryContext for an export request.-// SkipMasking has to be set here, not only checked around the post-execution-// MaskResults pass: queryRetry reads it off the QueryContext to decide-// maskingEnabled and whether to extract sensitive predicate columns, and both-// of those run before the export's own check. See PR #20487 review-// (RainbowDashy). No driver reads the field.+// queryRetry reads SkipMasking off the QueryContext to decide whether to mask+// the results and check sensitive predicate columns. No driver reads the field.func buildExportQueryContext(restriction *store.EffectiveQueryDataPolicy, userEmail string, schema *string, container string, skipMasking bool) db.QueryContext {qc := db.QueryContext{Limit: int(restriction.MaximumResultRows),@@ -1367,7 +1364,8 @@ func doExport(statements = []parserbase.Statement{{Text: request.Statement}}}- results, spans, duration, queryErr := queryRetry(+ // queryRetry masks the results.+ results, _, duration, queryErr := queryRetry(ctx,stores,user,@@ -1387,13 +1385,6 @@ func doExport(return nil, duration, queryErr}- if !skipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {- masker := NewQueryResultMasker(stores)- if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {- return nil, duration, err- }- }-files, err := buildExportFiles(ctx, stores, instance, database, results, request)if err != nil {return nil, duration, errdiff --git a/backend/api/v1/sql_service_export.go b/backend/api/v1/sql_service_export.goindex 6da16b9181..da4d9280f5 100644--- a/backend/api/v1/sql_service_export.go+++ b/backend/api/v1/sql_service_export.go@@ -129,7 +129,7 @@ func resolveSharePointExportTarget(ctx context.Context, newClient func(sharepoinif err != nil {return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Wrap(err, "invalid SharePoint configuration"))}- location, err := client.Resolve(ctx, destinationURL)+ location, err := client.ResolveRecent(ctx, destinationURL)if err != nil {if errors.Is(err, sharepoint.ErrNotFoundOrNoAccess) || errors.Is(err, sharepoint.ErrCredential) {return nil, connect.NewError(connect.CodeFailedPrecondition, err)diff --git a/backend/api/v1/sql_service_export_test.go b/backend/api/v1/sql_service_export_test.goindex f1fe060aa9..41089bae50 100644--- a/backend/api/v1/sql_service_export_test.go+++ b/backend/api/v1/sql_service_export_test.go@@ -280,6 +280,10 @@ func (c *exportSharePointClient) Resolve(_ context.Context, canonicalURL string)return c.location, nil}+func (c *exportSharePointClient) ResolveRecent(ctx context.Context, canonicalURL string) (*sharepoint.Location, error) {+ return c.Resolve(ctx, canonicalURL)+}+func (c *exportSharePointClient) Upload(_ context.Context, loc *sharepoint.Location, folderName string, files []sharepoint.File) (*sharepoint.Folder, error) {c.uploadedTo, c.uploadedFolder, c.uploadedFiles = loc, folderName, filesif c.uploadErr != nil {diff --git a/backend/plugin/sharepoint/client.go b/backend/plugin/sharepoint/client.goindex 78e6c7da60..347ed06a04 100644--- a/backend/plugin/sharepoint/client.go+++ b/backend/plugin/sharepoint/client.go@@ -9,26 +9,32 @@ import ("net/url""regexp""strings"+ "sync""time""github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/azcore/policy""github.com/Azure/azure-sdk-for-go/sdk/azidentity""github.com/pkg/errors"- "golang.org/x/sync/errgroup")const (graphBaseURL = "https://graph.microsoft.com/v1.0"graphScope = "https://graph.microsoft.com/.default"- // Graph requires upload chunks to be multiples of 320 KiB.- uploadChunkSize = 16 * 320 * 1024+ // Graph requires upload chunks to be multiples of 320 KiB and smaller than+ // 60 MiB. Each chunk pays a fixed round trip, so larger chunks upload+ // faster; chunks above 20 MiB measured no faster and resend more after a+ // stalled request.+ uploadChunkSize = 64 * 320 * 1024// Files up to this size upload in one request; larger files use a// resumable upload session.simpleUploadMaxSize = 4 << 20maxConcurrentUploads = 4maxAttempts = 5requestTimeout = 5 * time.Minute+ // recentResolveTTL bounds how long ResolveRecent reuses a resolution, so a+ // batch export resolves its destination once.+ recentResolveTTL = time.Minute)var aadstsPattern = regexp.MustCompile(`AADSTS\d+`)@@ -45,6 +51,17 @@ type graphClient struct {maxConcurrentUploads intmaxAttempts intwait func(ctx context.Context, d time.Duration) error+ // now is time.Now when nil.+ now func() time.Time++ recentMu sync.Mutex+ // recent maps canonical URLs to resolutions made within recentResolveTTL.+ recent map[string]recentLocation+}++type recentLocation struct {+ location Location+ expires time.Time}func newGraphClient(cfg Config) (Client, error) {@@ -119,6 +136,56 @@ func (c *graphClient) Resolve(ctx context.Context, canonicalURL string) (*Locatiif err != nil {return nil, err}+ return c.resolveCanonical(ctx, canonical)+}++func (c *graphClient) ResolveRecent(ctx context.Context, canonicalURL string) (*Location, error) {+ canonical, err := NormalizeURL(canonicalURL)+ if err != nil {+ return nil, err+ }+ c.recentMu.Lock()+ entry, ok := c.recent[canonical]+ c.recentMu.Unlock()+ if ok && c.clock().Before(entry.expires) {+ loc := entry.location+ loc.reused = true+ return &loc, nil+ }+ return c.resolveCanonical(ctx, canonical)+}++// resolveCanonical resolves a URL NormalizeURL returned and records the+// outcome for ResolveRecent.+func (c *graphClient) resolveCanonical(ctx context.Context, canonical string) (*Location, error) {+ loc, err := c.lookup(ctx, canonical)+ if err != nil {+ c.forget(canonical)+ return nil, err+ }+ c.recentMu.Lock()+ if c.recent == nil {+ c.recent = map[string]recentLocation{}+ }+ c.recent[canonical] = recentLocation{location: *loc, expires: c.clock().Add(recentResolveTTL)}+ c.recentMu.Unlock()+ return loc, nil+}++func (c *graphClient) forget(canonical string) {+ c.recentMu.Lock()+ defer c.recentMu.Unlock()+ delete(c.recent, canonical)+}++func (c *graphClient) clock() time.Time {+ if c.now != nil {+ return c.now()+ }+ return time.Now()+}++func (c *graphClient) lookup(ctx context.Context, canonical string) (*Location, error) {u, err := url.Parse(canonical)if err != nil {return nil, errors.Wrap(err, "invalid SharePoint URL")@@ -135,7 +202,7 @@ func (c *graphClient) Resolve(ctx context.Context, canonicalURL string) (*Locatiif item.ID == "" || item.ParentReference.DriveID == "" {return nil, errors.Errorf("Microsoft Graph did not return the document library of %s", canonical)}- return &Location{DriveID: item.ParentReference.DriveID, ItemID: item.ID, WebURL: item.WebURL}, nil+ return &Location{DriveID: item.ParentReference.DriveID, ItemID: item.ID, WebURL: item.WebURL, url: canonical}, nil}func (c *graphClient) Upload(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error) {@@ -154,65 +221,104 @@ func (c *graphClient) Upload(ctx context.Context, loc *Location, folderName striseen[key] = true}- folder, err := c.ensureFolder(ctx, loc, folderName)- if err != nil {- return nil, ¬hingUploadedError{err: err}- }- g, gctx := errgroup.WithContext(ctx)- g.SetLimit(max(c.maxConcurrentUploads, 1))- for _, file := range files {- g.Go(func() error {- return c.uploadFile(gctx, loc.DriveID, folder.ID, file)- })+ folder, err := c.uploadFiles(ctx, loc, folderName, files)+ if err == nil || !errors.Is(err, ErrNotFoundOrNoAccess) {+ return folder, err}- if err := g.Wait(); err != nil {+ c.forget(loc.url)+ if !loc.reused || !errors.Is(err, ErrNothingUploaded) {return nil, err}- return &Folder{Name: folderName, WebURL: folder.WebURL}, nil+ // The destination may have been deleted or recreated at the same URL since+ // the reused resolution.+ fresh, err := c.resolveCanonical(ctx, loc.url)+ if err != nil {+ return nil, ¬hingUploadedError{err: err}+ }+ return c.uploadFiles(ctx, fresh, folderName, files)}-func (c *graphClient) ensureFolder(ctx context.Context, loc *Location, folderName string) (*driveItem, error) {- parent := itemPath(loc.DriveID, loc.ItemID)- body := map[string]any{- "name": folderName,- "folder": map[string]any{},- "@microsoft.graph.conflictBehavior": "fail",- }- var folder driveItem- err := c.graphJSON(ctx, http.MethodPost, parent+"/children", body, &folder)- if statusOf(err) == http.StatusConflict {- folder = driveItem{}- err = c.graphJSON(ctx, http.MethodGet, parent+":/"+escapeSegment(folderName)+"?$select=id,webUrl,folder", nil, &folder)- if err == nil && folder.Folder == nil {- return nil, errors.Errorf("a file named %q already exists in the SharePoint destination", folderName)+// uploadFiles addresses files by path, so the first file to arrive creates the+// export folder.+func (c *graphClient) uploadFiles(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error) {+ folderPath := itemPath(loc.DriveID, loc.ItemID) + ":/" + escapeSegment(folderName) + "/"+ uploadCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ var (+ wg sync.WaitGroup+ mu sync.Mutex+ firstErr error+ // Graph creates nothing for a request it rejects, the folder included,+ // so nothing was uploaded when every attempted file was rejected. A+ // rejection lets the other uploads finish rather than canceling them,+ // because a canceled upload may have been committed.+ attempted, rejected int+ )+ slots := make(chan struct{}, max(c.maxConcurrentUploads, 1))+ for _, file := range files {+ slots <- struct{}{}+ mu.Lock()+ stop := firstErr != nil || uploadCtx.Err() != nil+ if !stop {+ attempted+++ }+ mu.Unlock()+ if stop {+ <-slots+ break}+ wg.Go(func() {+ defer func() { <-slots }()+ wasRejected, err := c.uploadFile(uploadCtx, folderPath, file)+ if err == nil {+ return+ }+ mu.Lock()+ defer mu.Unlock()+ if wasRejected {+ rejected+++ } else {+ cancel()+ }+ if firstErr == nil {+ firstErr = err+ }+ })}- if err != nil {- return nil, errors.Wrapf(err, "failed to create SharePoint folder %q", folderName)+ wg.Wait()+ if firstErr != nil {+ if rejected == attempted {+ return nil, ¬hingUploadedError{err: firstErr}+ }+ return nil, firstErr}- if folder.ID == "" {- return nil, errors.Errorf("Microsoft Graph did not return SharePoint folder %q", folderName)+ if attempted < len(files) {+ return nil, ctx.Err()}- return &folder, nil+ return &Folder{Name: folderName, WebURL: strings.TrimRight(loc.WebURL, "/") + "/" + url.PathEscape(folderName)}, nil}-func (c *graphClient) uploadFile(ctx context.Context, driveID, folderID string, file File) error {- err := c.putFile(ctx, driveID, folderID, file)+// uploadFile reports whether Graph rejected the file's upload as unauthorized,+// forbidden, or not found.+func (c *graphClient) uploadFile(ctx context.Context, folderPath string, file File) (bool, error) {+ err := c.putFile(ctx, folderPath, file)if err == nil {- return nil+ return false, nil}if statusOf(err) == http.StatusConflict {- return c.reconcile(ctx, driveID, folderID, file, false)+ return false, c.reconcile(ctx, folderPath, file, false)}var unknown *unknownOutcomeError- if errors.As(err, &unknown) {- return c.reconcile(ctx, driveID, folderID, file, true)+ var graphErr *graphError+ if errors.As(err, &unknown) || (errors.As(err, &graphErr) && graphErr.afterUnknownOutcome) {+ return false, c.reconcile(ctx, folderPath, file, true)}- return errors.Wrapf(err, "failed to upload %q", file.Name)+ rejected := errors.Is(err, ErrNotFoundOrNoAccess) || errors.Is(err, ErrCredential)+ return rejected, errors.Wrapf(err, "failed to upload %q", file.Name)}-func (c *graphClient) putFile(ctx context.Context, driveID, folderID string, file File) error {- target := itemPath(driveID, folderID) + ":/" + escapeSegment(file.Name) + ":"+func (c *graphClient) putFile(ctx context.Context, folderPath string, file File) error {+ target := folderPath + escapeSegment(file.Name) + ":"if len(file.Content) <= c.simpleUploadMaxSize {return c.graphRaw(ctx, http.MethodPut, target+"/[email protected]=fail", file.Content)}@@ -305,9 +411,9 @@ func nextExpectedOffset(body []byte, fallback int) (int, error) {// reconcile accepts an existing file as uploaded only if this application// created it with the same size; export folder names are unique per export.-func (c *graphClient) reconcile(ctx context.Context, driveID, folderID string, file File, outcomeUnknown bool) error {+func (c *graphClient) reconcile(ctx context.Context, folderPath string, file File, outcomeUnknown bool) error {var item driveItem- err := c.graphJSON(ctx, http.MethodGet, itemPath(driveID, folderID)+":/"+escapeSegment(file.Name)+"?$select=size,folder,createdBy", nil, &item)+ err := c.graphJSON(ctx, http.MethodGet, folderPath+escapeSegment(file.Name)+"?$select=size,folder,createdBy", nil, &item)if err == nil && item.Folder == nil && item.Size == int64(len(file.Content)) &&item.CreatedBy.Application != nil && strings.EqualFold(item.CreatedBy.Application.ID, c.clientID) {return nildiff --git a/backend/plugin/sharepoint/client_test.go b/backend/plugin/sharepoint/client_test.goindex 7ca6f5a521..d87c91d4ae 100644--- a/backend/plugin/sharepoint/client_test.go+++ b/backend/plugin/sharepoint/client_test.go@@ -91,7 +91,7 @@ func TestResolve(t *testing.T) {}))loc, err := client.Resolve(context.Background(), canonical)require.NoError(t, err)- require.Equal(t, &Location{DriveID: "d1", ItemID: "root1", WebURL: "https://contoso.sharepoint.com/sites/finance/Shared%20Documents"}, loc)+ require.Equal(t, &Location{DriveID: "d1", ItemID: "root1", WebURL: "https://contoso.sharepoint.com/sites/finance/Shared%20Documents", url: canonical}, loc)})t.Run("folder", func(t *testing.T) {@@ -100,7 +100,7 @@ func TestResolve(t *testing.T) {}))loc, err := client.Resolve(context.Background(), canonical)require.NoError(t, err)- require.Equal(t, &Location{DriveID: "d1", ItemID: "item1", WebURL: canonical}, loc)+ require.Equal(t, &Location{DriveID: "d1", ItemID: "item1", WebURL: canonical, url: canonical}, loc)})t.Run("file", func(t *testing.T) {@@ -138,8 +138,8 @@ type fakeSession struct {type fakeGraph struct {t *testing.Tmu sync.Mutex- // folderIDs maps folder IDs to names.- folderIDs map[string]string+ // folders holds the names of the folders under the destination.+ folders map[string]bool// files is keyed by "folderName/fileName".files map[string]fakeFilesessions map[string]*fakeSession@@ -153,16 +153,7 @@ type fakeGraph struct {}func newFakeGraph(t *testing.T) *fakeGraph {- return &fakeGraph{t: t, folderIDs: map[string]string{}, files: map[string]fakeFile{}, sessions: map[string]*fakeSession{}}-}--func (f *fakeGraph) folderID(name string) string {- for id, folderName := range f.folderIDs {- if folderName == name {- return id- }- }- return ""+ return &fakeGraph{t: t, folders: map[string]bool{}, files: map[string]fakeFile{}, sessions: map[string]*fakeSession{}}}func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {@@ -177,39 +168,16 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {return}require.Equal(f.t, "Bearer test-token", r.Header.Get("Authorization"))- rest, ok := strings.CutPrefix(r.URL.Path, "/drives/d1/items/")+ // Files are addressed by path under the destination item "dest".+ rest, ok := strings.CutPrefix(r.URL.Path, "/drives/d1/items/dest:/")if !ok {writeGraphError(w, http.StatusNotFound, "itemNotFound")return}- if rest == "dest/children" && r.Method == http.MethodPost {- var body map[string]any- require.NoError(f.t, json.NewDecoder(r.Body).Decode(&body))- require.Equal(f.t, "fail", body["@microsoft.graph.conflictBehavior"])- name, ok := body["name"].(string)- require.True(f.t, ok)- if f.folderID(name) != "" {- writeGraphError(w, http.StatusConflict, "nameAlreadyExists")- return- }- id := fmt.Sprintf("folder%d", len(f.folderIDs)+1)- f.folderIDs[id] = name- writeJSON(w, http.StatusCreated, map[string]any{"id": id, "name": name, "webUrl": "https://sp/" + id, "folder": map[string]any{}})- return- }- itemID, sub, _ := strings.Cut(rest, ":/")- if itemID == "dest" && r.Method == http.MethodGet {- id := f.folderID(sub)- if id == "" {- writeGraphError(w, http.StatusNotFound, "itemNotFound")- return- }- writeJSON(w, http.StatusOK, map[string]any{"id": id, "webUrl": "https://sp/" + id, "folder": map[string]any{}})- return- }- folderName, ok := f.folderIDs[itemID]+ folderName, sub, ok := strings.Cut(rest, "/")if !ok {- writeGraphError(w, http.StatusNotFound, "itemNotFound")+ f.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)+ writeGraphError(w, http.StatusBadRequest, "invalidRequest")return}switch {@@ -219,6 +187,7 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {writeGraphError(w, http.StatusConflict, "nameAlreadyExists")return}+ f.folders[folderName] = truef.sessionCount++id := fmt.Sprint(f.sessionCount)f.sessions[id] = &fakeSession{key: key}@@ -230,6 +199,7 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {return}data, _ := io.ReadAll(r.Body)+ f.folders[folderName] = truef.files[key] = fakeFile{content: data, appID: testAppID}writeJSON(w, http.StatusCreated, map[string]any{"id": key})case r.Method == http.MethodGet:@@ -290,7 +260,7 @@ func (f *fakeGraph) serveUpload(w http.ResponseWriter, r *http.Request) {const testFolder = "20261022T010203Z [email protected] db 42"var (- testLocation = &Location{DriveID: "d1", ItemID: "dest"}+ testLocation = &Location{DriveID: "d1", ItemID: "dest", WebURL: "https://sp/dest"}testFiles = []File{{Name: "statement.sql", Content: []byte("SELECT 1;\n")},{Name: "result #1.csv", Content: []byte("a,b\n1,2\n")},@@ -307,12 +277,14 @@ func requireUploaded(t *testing.T, fake *fakeGraph) {}}+const testFolderURL = "https://sp/dest/20261022T010203Z%[email protected]%20db%2042"+func TestUpload(t *testing.T) {fake := newFakeGraph(t)client, waits := newTestClient(t, fake)folder, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)require.NoError(t, err)- require.Equal(t, &Folder{Name: testFolder, WebURL: "https://sp/folder1"}, folder)+ require.Equal(t, &Folder{Name: testFolder, WebURL: testFolderURL}, folder)requireUploaded(t, fake)require.Equal(t, []string{"bytes 0-3/10", "bytes 4-7/10", "bytes 8-9/10",@@ -323,21 +295,11 @@ func TestUpload(t *testing.T) {// Retrying the same export reuses the folder and the uploaded files.folder, err = client.Upload(context.Background(), testLocation, testFolder, testFiles)require.NoError(t, err)- require.Equal(t, &Folder{Name: testFolder, WebURL: "https://sp/folder1"}, folder)- require.Len(t, fake.folderIDs, 1)+ require.Equal(t, &Folder{Name: testFolder, WebURL: testFolderURL}, folder)+ require.Len(t, fake.folders, 1)require.Equal(t, 2, fake.sessionCount)}-func TestUploadReusesExistingFolder(t *testing.T) {- fake := newFakeGraph(t)- fake.folderIDs["existing"] = testFolder- client, _ := newTestClient(t, fake)- folder, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)- require.NoError(t, err)- require.Equal(t, "https://sp/existing", folder.WebURL)- requireUploaded(t, fake)-}-func TestUploadFileConflict(t *testing.T) {files := []File{{Name: "result.csv", Content: []byte("a,b\n")}}tests := []struct {@@ -352,7 +314,7 @@ func TestUploadFileConflict(t *testing.T) {for _, tc := range tests {t.Run(tc.name, func(t *testing.T) {fake := newFakeGraph(t)- fake.folderIDs["existing"] = testFolder+ fake.folders[testFolder] = truefake.files[testFolder+"/result.csv"] = tc.existingclient, _ := newTestClient(t, fake)_, err := client.Upload(context.Background(), testLocation, testFolder, files)@@ -368,27 +330,11 @@ func TestUploadFileConflict(t *testing.T) {}}-func TestUploadFolderCreationFailureUploadsNothing(t *testing.T) {- fake := newFakeGraph(t)- fake.override = func(w http.ResponseWriter, r *http.Request) bool {- if r.URL.Path != "/drives/d1/items/dest/children" {- return false- }- writeGraphError(w, http.StatusForbidden, "accessDenied")- return true- }- client, _ := newTestClient(t, fake)- _, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)- require.ErrorIs(t, err, ErrNothingUploaded)- require.ErrorIs(t, err, ErrNotFoundOrNoAccess)- require.Zero(t, fake.sessionCount)-}-func TestUploadHonorsRetryAfter(t *testing.T) {fake := newFakeGraph(t)throttled := false- fake.override = func(w http.ResponseWriter, r *http.Request) bool {- if throttled || r.URL.Path != "/drives/d1/items/dest/children" {+ fake.override = func(w http.ResponseWriter, _ *http.Request) bool {+ if throttled {return false}throttled = true@@ -435,12 +381,138 @@ func TestUploadUnconfirmed(t *testing.T) {require.Equal(t, []time.Duration{time.Second, 2 * time.Second}, *waits)}-func TestUploadForbidden(t *testing.T) {- client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+func TestUploadForbiddenUploadsNothing(t *testing.T) {+ fake := newFakeGraph(t)+ fake.override = func(w http.ResponseWriter, _ *http.Request) bool {writeGraphError(w, http.StatusForbidden, "accessDenied")- }))+ return true+ }+ client, _ := newTestClient(t, fake)+ client.maxConcurrentUploads = len(testFiles)_, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)require.ErrorIs(t, err, ErrNotFoundOrNoAccess)+ require.ErrorIs(t, err, ErrNothingUploaded)+ require.Empty(t, fake.folders)+}++func TestUploadRejectionDoesNotDenyAConcurrentUpload(t *testing.T) {+ files := []File{{Name: "a.sql", Content: []byte("a")}, {Name: "b.csv", Content: []byte("b")}}+ rejected := make(chan struct{})+ client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.Contains(r.URL.Path, "b.csv") {+ writeGraphError(w, http.StatusForbidden, "accessDenied")+ close(rejected)+ return+ }+ <-rejected+ writeJSON(w, http.StatusCreated, map[string]any{"id": "a"})+ }))+ client.simpleUploadMaxSize = simpleUploadMaxSize+ client.maxConcurrentUploads = len(files)+ _, err := client.Upload(context.Background(), testLocation, testFolder, files)+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)+ require.NotErrorIs(t, err, ErrNothingUploaded)+}++// recreatableDestination serves a destination URL that resolves to the item+// named by current, under drive d1. Only the item "dest" holds files.+type recreatableDestination struct {+ fake *fakeGraph+ current string+ resolve int+}++func newRecreatableDestination(t *testing.T, current string) *recreatableDestination {+ d := &recreatableDestination{fake: newFakeGraph(t), current: current}+ d.fake.override = func(w http.ResponseWriter, r *http.Request) bool {+ if !strings.HasPrefix(r.URL.Path, "/shares/") {+ return false+ }+ d.resolve+++ if d.current == "" {+ writeGraphError(w, http.StatusForbidden, "accessDenied")+ return true+ }+ writeJSON(w, http.StatusOK, map[string]any{+ "id": d.current, "webUrl": "https://sp/" + d.current, "folder": map[string]any{},+ "parentReference": map[string]any{"driveId": "d1"},+ })+ return true+ }+ return d+}++func TestResolveRecent(t *testing.T) {+ dest := newRecreatableDestination(t, "dest")+ client, _ := newTestClient(t, dest.fake)+ now := time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)+ client.now = func() time.Time { return now }+ ctx := context.Background()++ loc, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, "dest", loc.ItemID)+ _, err = client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, 1, dest.resolve)++ // Resolve always asks Graph.+ _, err = client.Resolve(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, 2, dest.resolve)++ now = now.Add(recentResolveTTL)+ _, err = client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, 3, dest.resolve)+}++func TestUploadResolvesReusedDestinationAgain(t *testing.T) {+ ctx := context.Background()+ t.Run("recreated destination receives the export", func(t *testing.T) {+ dest := newRecreatableDestination(t, "deleted")+ client, _ := newTestClient(t, dest.fake)+ _, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ dest.current = "dest"+ loc, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, "deleted", loc.ItemID)++ folder, err := client.Upload(ctx, loc, testFolder, testFiles)+ require.NoError(t, err)+ require.Equal(t, testFolderURL, folder.WebURL)+ requireUploaded(t, dest.fake)+ require.Equal(t, 2, dest.resolve)+ })+ t.Run("deleted destination fails without uploading", func(t *testing.T) {+ dest := newRecreatableDestination(t, "deleted")+ client, _ := newTestClient(t, dest.fake)+ _, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ dest.current = ""+ loc, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)++ _, err = client.Upload(ctx, loc, testFolder, testFiles)+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)+ require.ErrorIs(t, err, ErrNothingUploaded)+ require.Equal(t, 2, dest.resolve)+ })+ t.Run("fresh resolution is not resolved again", func(t *testing.T) {+ dest := newRecreatableDestination(t, "deleted")+ client, _ := newTestClient(t, dest.fake)+ loc, err := client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)++ _, err = client.Upload(ctx, loc, testFolder, testFiles)+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)+ require.Equal(t, 1, dest.resolve)+ // The failure drops the resolution, so the next export asks Graph.+ _, err = client.ResolveRecent(ctx, resolveShareURL)+ require.NoError(t, err)+ require.Equal(t, 2, dest.resolve)+ })}func TestTransportErrorOmitsURL(t *testing.T) {@@ -460,8 +532,8 @@ func TestUploadSmallFilesInOneRequestEach(t *testing.T) {require.NoError(t, err)requireUploaded(t, fake)require.Zero(t, fake.sessionCount)- // One request creates the folder and one uploads each file.- require.Equal(t, 1+len(testFiles), fake.requestCount)+ // One request uploads each file; the first also creates the folder.+ require.Equal(t, len(testFiles), fake.requestCount)}func TestUploadInParallel(t *testing.T) {@@ -470,10 +542,6 @@ func TestUploadInParallel(t *testing.T) {inFlight := 0allInFlight := make(chan struct{})client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if r.Method == http.MethodPost {- writeJSON(w, http.StatusCreated, map[string]any{"id": "folder1", "webUrl": "https://sp/folder1", "folder": map[string]any{}})- return- }mu.Lock()inFlight++if inFlight == len(files) {@@ -492,3 +560,25 @@ func TestUploadInParallel(t *testing.T) {_, err := client.Upload(context.Background(), testLocation, testFolder, files)require.NoError(t, err)}++func TestUploadRejectedAfterLostResponseIsUnconfirmed(t *testing.T) {+ puts := 0+ client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method == http.MethodPut {+ puts+++ if puts == 1 {+ // The first attempt is committed but its response is lost.+ conn, _, err := w.(http.Hijacker).Hijack()+ require.NoError(t, err)+ _ = conn.Close()+ return+ }+ }+ writeGraphError(w, http.StatusForbidden, "accessDenied")+ }))+ client.simpleUploadMaxSize = simpleUploadMaxSize+ _, err := client.Upload(context.Background(), testLocation, testFolder, []File{{Name: "a.sql", Content: []byte("a")}})+ require.ErrorIs(t, err, ErrUnconfirmed)+ require.NotErrorIs(t, err, ErrNothingUploaded)+ require.Equal(t, 2, puts)+}diff --git a/backend/plugin/sharepoint/sharepoint.go b/backend/plugin/sharepoint/sharepoint.goindex c239acbd2c..a63caee420 100644--- a/backend/plugin/sharepoint/sharepoint.go+++ b/backend/plugin/sharepoint/sharepoint.go@@ -21,6 +21,11 @@ type Location struct {DriveID stringItemID stringWebURL string++ // url is the canonical URL the location was resolved from.+ url string+ // reused reports whether ResolveRecent returned an earlier resolution.+ reused bool}// File is one file of an export.@@ -60,9 +65,14 @@ type Client interface {// Resolve resolves a canonical URL from NormalizeURL to a library root or// folder. Resolving proves only that the destination is readable.Resolve(ctx context.Context, canonicalURL string) (*Location, error)+ // ResolveRecent is Resolve, except that it reuses a resolution of the same+ // URL made within the last minute.+ ResolveRecent(ctx context.Context, canonicalURL string) (*Location, error)// Upload creates folderName under loc and uploads files into it. Retrying// Upload with the same arguments reuses the folder and files already- // uploaded by the application instead of creating copies.+ // uploaded by the application instead of creating copies. If loc is a+ // reused resolution that Graph no longer finds, Upload resolves its URL+ // again and retries once.Upload(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error)}diff --git a/backend/plugin/sharepoint/transport.go b/backend/plugin/sharepoint/transport.goindex 5402d15949..e2cefa1d86 100644--- a/backend/plugin/sharepoint/transport.go+++ b/backend/plugin/sharepoint/transport.go@@ -24,6 +24,9 @@ type response struct {status intheader http.Headerbody []byte+ // afterUnknownOutcome reports that an earlier attempt of the request may+ // have taken effect.+ afterUnknownOutcome bool}// unknownOutcomeError means a request may or may not have taken effect.@@ -43,6 +46,9 @@ type graphError struct {status intcode stringmessage string+ // afterUnknownOutcome reports that an earlier attempt of the request may+ // have taken effect, so the error does not prove the request did nothing.+ afterUnknownOutcome bool}func newGraphError(resp *response) *graphError {@@ -53,7 +59,7 @@ func newGraphError(resp *response) *graphError {} `json:"error"`}_ = json.Unmarshal(resp.body, &body)- return &graphError{status: resp.status, code: body.Error.Code, message: body.Error.Message}+ return &graphError{status: resp.status, code: body.Error.Code, message: body.Error.Message, afterUnknownOutcome: resp.afterUnknownOutcome}}func (e *graphError) sentinel() error {@@ -161,6 +167,7 @@ func (c *graphClient) send(ctx context.Context, method, target string, body []byoutcomeUnknown = truelastErr = newGraphError(resp)default:+ resp.afterUnknownOutcome = outcomeUnknownreturn resp, nil}if attempt >= c.maxAttempts {diff --git a/backend/tests/sql_export_masking_test.go b/backend/tests/sql_export_masking_test.gonew file mode 100644index 0000000000..e7956f1d4d--- /dev/null+++ b/backend/tests/sql_export_masking_test.go@@ -0,0 +1,120 @@+package tests++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "strings"+ "testing"++ "connectrpc.com/connect"+ "github.com/alexmullins/zip"+ "github.com/stretchr/testify/require"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// TestSQLExportMatchesQueryMasking checks that an export masks a column the+// same way a query does. MD5 masking is not idempotent, so masking a result+// twice changes the exported value.+func TestSQLExportMatchesQueryMasking(t *testing.T) {+ t.Parallel()+ a := require.New(t)+ ctl, ctx := startWorkspace(context.Background(), t)++ _, err := ctl.settingServiceClient.UpdateSetting(ctx, connect.NewRequest(&v1pb.UpdateSettingRequest{+ AllowMissing: true,+ Setting: &v1pb.Setting{+ Name: "settings/" + v1pb.Setting_SEMANTIC_TYPES.String(),+ Value: &v1pb.SettingValue{+ Value: &v1pb.SettingValue_SemanticType{+ SemanticType: &v1pb.SemanticTypeSetting{+ Types: []*v1pb.SemanticTypeSetting_SemanticType{{+ Id: "hashed",+ Title: "Hashed",+ Algorithm: &v1pb.Algorithm{+ Mask: &v1pb.Algorithm_Md5Mask{Md5Mask: &v1pb.Algorithm_MD5Mask{Salt: "salt"}},+ },+ }},+ },+ },+ },+ },+ }))+ a.NoError(err)++ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("export-masking"),+ Instance: &v1pb.Instance{+ SyncDatabases: &v1pb.SyncDatabases{},+ Title: "Export masking",+ Engine: v1pb.Engine_POSTGRES,+ Environment: new("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{sharedPgTarget(t).adminDataSource()},+ },+ }))+ a.NoError(err)+ databaseName := generateRandomString("maskdb")+ a.NoError(ctl.createDatabase(ctx, ctl.project, instanceResp.Msg, nil, databaseName, ""))+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instanceResp.Msg.Name, databaseName),+ }))+ a.NoError(err)+ database := databaseResp.Msg++ setup, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: ctl.project.Name,+ Sheet: &v1pb.Sheet{Content: []byte("CREATE TABLE employee(id INT PRIMARY KEY, secret TEXT); INSERT INTO employee VALUES (1, 'plain');")},+ }))+ a.NoError(err)+ a.NoError(ctl.changeDatabase(ctx, ctl.project, database, setup.Msg, false))+ _, err = ctl.databaseCatalogServiceClient.UpdateDatabaseCatalog(ctx, connect.NewRequest(&v1pb.UpdateDatabaseCatalogRequest{+ Catalog: &v1pb.DatabaseCatalog{+ Name: database.Name + "/catalog",+ Schemas: []*v1pb.SchemaCatalog{{+ Name: "public",+ Tables: []*v1pb.TableCatalog{{+ Name: "employee",+ Kind: &v1pb.TableCatalog_Columns_{Columns: &v1pb.TableCatalog_Columns{+ Columns: []*v1pb.ColumnCatalog{{Name: "secret", SemanticType: "hashed"}},+ }},+ }},+ }},+ },+ }))+ a.NoError(err)++ const statement = "SELECT secret FROM employee"+ queryResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{Name: database.Name, Statement: statement}))+ a.NoError(err)+ a.Len(queryResp.Msg.Results, 1)+ a.Empty(queryResp.Msg.Results[0].Error)+ a.Len(queryResp.Msg.Results[0].Rows, 1)+ queried := queryResp.Msg.Results[0].Rows[0].Values[0].GetStringValue()+ a.NotEqual("plain", queried, "precondition: the semantic type must mask the column")++ exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(&v1pb.ExportRequest{+ Name: database.Name,+ Format: v1pb.ExportFormat_CSV,+ Statement: statement,+ }))+ a.NoError(err)+ content := exportResp.Msg.Content+ zipReader, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))+ a.NoError(err)+ var exported string+ for _, file := range zipReader.File {+ if !strings.HasSuffix(file.Name, ".result.csv") {+ continue+ }+ r, err := file.Open()+ a.NoError(err)+ data, err := io.ReadAll(r)+ a.NoError(err)+ exported = string(data)+ }+ // CSV export quotes text values.+ a.Equal("secret\n\""+queried+"\"", exported)+}diff --git a/docs/adr/0007-sharepoint-export-ownership-boundary.md b/docs/adr/0007-sharepoint-export-ownership-boundary.mdindex e0c2ddcff8..bca1309b23 100644--- a/docs/adr/0007-sharepoint-export-ownership-boundary.md+++ b/docs/adr/0007-sharepoint-export-ownership-boundary.md@@ -111,9 +111,10 @@ the same export file names. If delivery remains uncertain, Bytebase reports thatdelivery could not be confirmed and files may exist in SharePoint. It doesnot claim that no file was created.-Bytebase checks the destination once, before the query runs. A change made-while an export is running applies to the next export. A rejected destination-is never redirected to another one.+Bytebase checks the destination once, before the query runs, reusing a check of+the same URL made within the last minute so a batch export resolves its+destination once. A change made while an export is running applies to the next+export. A rejected destination is never redirected to another one.Internal upload retries reuse the generated files, file names, and captureddestination without rerunning the query. A new Export action starts a new export@@ -138,9 +139,13 @@ URL that does not resolve. It stores and returns only the canonical URL; resolvenames go stale on rename and are not kept. A SharePoint browsing picker isoutside the initial scope.-Bytebase resolves the stored URL at each export rather than pinning item IDs.-Renaming or moving the folder makes exports fail as not found; a folder-recreated at the same URL receives subsequent exports.+Bytebase resolves the stored URL rather than pinning item IDs, and reuses a+resolution for at most a minute. Renaming or moving the folder makes exports+fail as not found, except that an export reusing a resolution from before the+rename uploads into the renamed folder and returns a link to the old path. When+SharePoint rejects an upload into a reused resolution as not found, Bytebase+resolves the URL again and retries once, so a folder recreated at the same URL+receives that export.Customers create and rotate Entra client secrets. Expired or invalid credentialscause affected exports to fail with a clear credential error. An authorized user