Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

perf(export): mask exports once and cut SharePoint export round trips (#21868)

p0ny2f1c7be0ce94
diff --git a/backend/api/v1/project_service_export_setting_test.go b/backend/api/v1/project_service_export_setting_test.go
index 4a4ca45177..cdde45a394 100644
--- a/backend/api/v1/project_service_export_setting_test.go
+++ b/backend/api/v1/project_service_export_setting_test.go
@@ -29,6 +29,10 @@ func (c *fakeSharePointResolver) Resolve(_ context.Context, canonicalURL string)
return c.location, c.err
}
+func (*fakeSharePointResolver) ResolveRecent(context.Context, string) (*sharepoint.Location, error) {
+ return nil, errors.New("not used")
+}
+
func (*fakeSharePointResolver) Upload(context.Context, *sharepoint.Location, string, []sharepoint.File) (*sharepoint.Folder, error) {
return nil, errors.New("not used")
}
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index 95a239ec00..592135ee45 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -171,11 +171,8 @@ func (s *SQLService) AdminExecute(ctx context.Context, stream *connect.BidiStrea
}
// buildExportQueryContext assembles the db.QueryContext for an export request.
-// SkipMasking has to be set here, not only checked around the post-execution
-// MaskResults pass: queryRetry reads it off the QueryContext to decide
-// maskingEnabled and whether to extract sensitive predicate columns, and both
-// of those run before the export's own check. See PR #20487 review
-// (RainbowDashy). No driver reads the field.
+// queryRetry reads SkipMasking off the QueryContext to decide whether to mask
+// the results and check sensitive predicate columns. No driver reads the field.
func buildExportQueryContext(restriction *store.EffectiveQueryDataPolicy, userEmail string, schema *string, container string, skipMasking bool) db.QueryContext {
qc := db.QueryContext{
Limit: int(restriction.MaximumResultRows),
@@ -1367,7 +1364,8 @@ func doExport(
statements = []parserbase.Statement{{Text: request.Statement}}
}
- results, spans, duration, queryErr := queryRetry(
+ // queryRetry masks the results.
+ results, _, duration, queryErr := queryRetry(
ctx,
stores,
user,
@@ -1387,13 +1385,6 @@ func doExport(
return nil, duration, queryErr
}
- if !skipMasking && licenseService.IsFeatureEnabledForInstance(ctx, common.GetWorkspaceIDFromContext(ctx), v1pb.PlanFeature_FEATURE_DATA_MASKING, instance) == nil {
- masker := NewQueryResultMasker(stores)
- if err := masker.MaskResults(ctx, spans, results, instance, user); err != nil {
- return nil, duration, err
- }
- }
-
files, err := buildExportFiles(ctx, stores, instance, database, results, request)
if err != nil {
return nil, duration, err
diff --git a/backend/api/v1/sql_service_export.go b/backend/api/v1/sql_service_export.go
index 6da16b9181..da4d9280f5 100644
--- a/backend/api/v1/sql_service_export.go
+++ b/backend/api/v1/sql_service_export.go
@@ -129,7 +129,7 @@ func resolveSharePointExportTarget(ctx context.Context, newClient func(sharepoin
if err != nil {
return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Wrap(err, "invalid SharePoint configuration"))
}
- location, err := client.Resolve(ctx, destinationURL)
+ location, err := client.ResolveRecent(ctx, destinationURL)
if err != nil {
if errors.Is(err, sharepoint.ErrNotFoundOrNoAccess) || errors.Is(err, sharepoint.ErrCredential) {
return nil, connect.NewError(connect.CodeFailedPrecondition, err)
diff --git a/backend/api/v1/sql_service_export_test.go b/backend/api/v1/sql_service_export_test.go
index f1fe060aa9..41089bae50 100644
--- a/backend/api/v1/sql_service_export_test.go
+++ b/backend/api/v1/sql_service_export_test.go
@@ -280,6 +280,10 @@ func (c *exportSharePointClient) Resolve(_ context.Context, canonicalURL string)
return c.location, nil
}
+func (c *exportSharePointClient) ResolveRecent(ctx context.Context, canonicalURL string) (*sharepoint.Location, error) {
+ return c.Resolve(ctx, canonicalURL)
+}
+
func (c *exportSharePointClient) Upload(_ context.Context, loc *sharepoint.Location, folderName string, files []sharepoint.File) (*sharepoint.Folder, error) {
c.uploadedTo, c.uploadedFolder, c.uploadedFiles = loc, folderName, files
if c.uploadErr != nil {
diff --git a/backend/plugin/sharepoint/client.go b/backend/plugin/sharepoint/client.go
index 78e6c7da60..347ed06a04 100644
--- a/backend/plugin/sharepoint/client.go
+++ b/backend/plugin/sharepoint/client.go
@@ -9,26 +9,32 @@ import (
"net/url"
"regexp"
"strings"
+ "sync"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/pkg/errors"
- "golang.org/x/sync/errgroup"
)
const (
graphBaseURL = "https://graph.microsoft.com/v1.0"
graphScope = "https://graph.microsoft.com/.default"
- // Graph requires upload chunks to be multiples of 320 KiB.
- uploadChunkSize = 16 * 320 * 1024
+ // Graph requires upload chunks to be multiples of 320 KiB and smaller than
+ // 60 MiB. Each chunk pays a fixed round trip, so larger chunks upload
+ // faster; chunks above 20 MiB measured no faster and resend more after a
+ // stalled request.
+ uploadChunkSize = 64 * 320 * 1024
// Files up to this size upload in one request; larger files use a
// resumable upload session.
simpleUploadMaxSize = 4 << 20
maxConcurrentUploads = 4
maxAttempts = 5
requestTimeout = 5 * time.Minute
+ // recentResolveTTL bounds how long ResolveRecent reuses a resolution, so a
+ // batch export resolves its destination once.
+ recentResolveTTL = time.Minute
)
var aadstsPattern = regexp.MustCompile(`AADSTS\d+`)
@@ -45,6 +51,17 @@ type graphClient struct {
maxConcurrentUploads int
maxAttempts int
wait func(ctx context.Context, d time.Duration) error
+ // now is time.Now when nil.
+ now func() time.Time
+
+ recentMu sync.Mutex
+ // recent maps canonical URLs to resolutions made within recentResolveTTL.
+ recent map[string]recentLocation
+}
+
+type recentLocation struct {
+ location Location
+ expires time.Time
}
func newGraphClient(cfg Config) (Client, error) {
@@ -119,6 +136,56 @@ func (c *graphClient) Resolve(ctx context.Context, canonicalURL string) (*Locati
if err != nil {
return nil, err
}
+ return c.resolveCanonical(ctx, canonical)
+}
+
+func (c *graphClient) ResolveRecent(ctx context.Context, canonicalURL string) (*Location, error) {
+ canonical, err := NormalizeURL(canonicalURL)
+ if err != nil {
+ return nil, err
+ }
+ c.recentMu.Lock()
+ entry, ok := c.recent[canonical]
+ c.recentMu.Unlock()
+ if ok && c.clock().Before(entry.expires) {
+ loc := entry.location
+ loc.reused = true
+ return &loc, nil
+ }
+ return c.resolveCanonical(ctx, canonical)
+}
+
+// resolveCanonical resolves a URL NormalizeURL returned and records the
+// outcome for ResolveRecent.
+func (c *graphClient) resolveCanonical(ctx context.Context, canonical string) (*Location, error) {
+ loc, err := c.lookup(ctx, canonical)
+ if err != nil {
+ c.forget(canonical)
+ return nil, err
+ }
+ c.recentMu.Lock()
+ if c.recent == nil {
+ c.recent = map[string]recentLocation{}
+ }
+ c.recent[canonical] = recentLocation{location: *loc, expires: c.clock().Add(recentResolveTTL)}
+ c.recentMu.Unlock()
+ return loc, nil
+}
+
+func (c *graphClient) forget(canonical string) {
+ c.recentMu.Lock()
+ defer c.recentMu.Unlock()
+ delete(c.recent, canonical)
+}
+
+func (c *graphClient) clock() time.Time {
+ if c.now != nil {
+ return c.now()
+ }
+ return time.Now()
+}
+
+func (c *graphClient) lookup(ctx context.Context, canonical string) (*Location, error) {
u, err := url.Parse(canonical)
if err != nil {
return nil, errors.Wrap(err, "invalid SharePoint URL")
@@ -135,7 +202,7 @@ func (c *graphClient) Resolve(ctx context.Context, canonicalURL string) (*Locati
if item.ID == "" || item.ParentReference.DriveID == "" {
return nil, errors.Errorf("Microsoft Graph did not return the document library of %s", canonical)
}
- return &Location{DriveID: item.ParentReference.DriveID, ItemID: item.ID, WebURL: item.WebURL}, nil
+ return &Location{DriveID: item.ParentReference.DriveID, ItemID: item.ID, WebURL: item.WebURL, url: canonical}, nil
}
func (c *graphClient) Upload(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error) {
@@ -154,65 +221,104 @@ func (c *graphClient) Upload(ctx context.Context, loc *Location, folderName stri
seen[key] = true
}
- folder, err := c.ensureFolder(ctx, loc, folderName)
- if err != nil {
- return nil, &nothingUploadedError{err: err}
- }
- g, gctx := errgroup.WithContext(ctx)
- g.SetLimit(max(c.maxConcurrentUploads, 1))
- for _, file := range files {
- g.Go(func() error {
- return c.uploadFile(gctx, loc.DriveID, folder.ID, file)
- })
+ folder, err := c.uploadFiles(ctx, loc, folderName, files)
+ if err == nil || !errors.Is(err, ErrNotFoundOrNoAccess) {
+ return folder, err
}
- if err := g.Wait(); err != nil {
+ c.forget(loc.url)
+ if !loc.reused || !errors.Is(err, ErrNothingUploaded) {
return nil, err
}
- return &Folder{Name: folderName, WebURL: folder.WebURL}, nil
+ // The destination may have been deleted or recreated at the same URL since
+ // the reused resolution.
+ fresh, err := c.resolveCanonical(ctx, loc.url)
+ if err != nil {
+ return nil, &nothingUploadedError{err: err}
+ }
+ return c.uploadFiles(ctx, fresh, folderName, files)
}
-func (c *graphClient) ensureFolder(ctx context.Context, loc *Location, folderName string) (*driveItem, error) {
- parent := itemPath(loc.DriveID, loc.ItemID)
- body := map[string]any{
- "name": folderName,
- "folder": map[string]any{},
- "@microsoft.graph.conflictBehavior": "fail",
- }
- var folder driveItem
- err := c.graphJSON(ctx, http.MethodPost, parent+"/children", body, &folder)
- if statusOf(err) == http.StatusConflict {
- folder = driveItem{}
- err = c.graphJSON(ctx, http.MethodGet, parent+":/"+escapeSegment(folderName)+"?$select=id,webUrl,folder", nil, &folder)
- if err == nil && folder.Folder == nil {
- return nil, errors.Errorf("a file named %q already exists in the SharePoint destination", folderName)
+// uploadFiles addresses files by path, so the first file to arrive creates the
+// export folder.
+func (c *graphClient) uploadFiles(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error) {
+ folderPath := itemPath(loc.DriveID, loc.ItemID) + ":/" + escapeSegment(folderName) + "/"
+ uploadCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ var (
+ wg sync.WaitGroup
+ mu sync.Mutex
+ firstErr error
+ // Graph creates nothing for a request it rejects, the folder included,
+ // so nothing was uploaded when every attempted file was rejected. A
+ // rejection lets the other uploads finish rather than canceling them,
+ // because a canceled upload may have been committed.
+ attempted, rejected int
+ )
+ slots := make(chan struct{}, max(c.maxConcurrentUploads, 1))
+ for _, file := range files {
+ slots <- struct{}{}
+ mu.Lock()
+ stop := firstErr != nil || uploadCtx.Err() != nil
+ if !stop {
+ attempted++
+ }
+ mu.Unlock()
+ if stop {
+ <-slots
+ break
}
+ wg.Go(func() {
+ defer func() { <-slots }()
+ wasRejected, err := c.uploadFile(uploadCtx, folderPath, file)
+ if err == nil {
+ return
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if wasRejected {
+ rejected++
+ } else {
+ cancel()
+ }
+ if firstErr == nil {
+ firstErr = err
+ }
+ })
}
- if err != nil {
- return nil, errors.Wrapf(err, "failed to create SharePoint folder %q", folderName)
+ wg.Wait()
+ if firstErr != nil {
+ if rejected == attempted {
+ return nil, &nothingUploadedError{err: firstErr}
+ }
+ return nil, firstErr
}
- if folder.ID == "" {
- return nil, errors.Errorf("Microsoft Graph did not return SharePoint folder %q", folderName)
+ if attempted < len(files) {
+ return nil, ctx.Err()
}
- return &folder, nil
+ return &Folder{Name: folderName, WebURL: strings.TrimRight(loc.WebURL, "/") + "/" + url.PathEscape(folderName)}, nil
}
-func (c *graphClient) uploadFile(ctx context.Context, driveID, folderID string, file File) error {
- err := c.putFile(ctx, driveID, folderID, file)
+// uploadFile reports whether Graph rejected the file's upload as unauthorized,
+// forbidden, or not found.
+func (c *graphClient) uploadFile(ctx context.Context, folderPath string, file File) (bool, error) {
+ err := c.putFile(ctx, folderPath, file)
if err == nil {
- return nil
+ return false, nil
}
if statusOf(err) == http.StatusConflict {
- return c.reconcile(ctx, driveID, folderID, file, false)
+ return false, c.reconcile(ctx, folderPath, file, false)
}
var unknown *unknownOutcomeError
- if errors.As(err, &unknown) {
- return c.reconcile(ctx, driveID, folderID, file, true)
+ var graphErr *graphError
+ if errors.As(err, &unknown) || (errors.As(err, &graphErr) && graphErr.afterUnknownOutcome) {
+ return false, c.reconcile(ctx, folderPath, file, true)
}
- return errors.Wrapf(err, "failed to upload %q", file.Name)
+ rejected := errors.Is(err, ErrNotFoundOrNoAccess) || errors.Is(err, ErrCredential)
+ return rejected, errors.Wrapf(err, "failed to upload %q", file.Name)
}
-func (c *graphClient) putFile(ctx context.Context, driveID, folderID string, file File) error {
- target := itemPath(driveID, folderID) + ":/" + escapeSegment(file.Name) + ":"
+func (c *graphClient) putFile(ctx context.Context, folderPath string, file File) error {
+ target := folderPath + escapeSegment(file.Name) + ":"
if len(file.Content) <= c.simpleUploadMaxSize {
return c.graphRaw(ctx, http.MethodPut, target+"/[email protected]=fail", file.Content)
}
@@ -305,9 +411,9 @@ func nextExpectedOffset(body []byte, fallback int) (int, error) {
// reconcile accepts an existing file as uploaded only if this application
// created it with the same size; export folder names are unique per export.
-func (c *graphClient) reconcile(ctx context.Context, driveID, folderID string, file File, outcomeUnknown bool) error {
+func (c *graphClient) reconcile(ctx context.Context, folderPath string, file File, outcomeUnknown bool) error {
var item driveItem
- err := c.graphJSON(ctx, http.MethodGet, itemPath(driveID, folderID)+":/"+escapeSegment(file.Name)+"?$select=size,folder,createdBy", nil, &item)
+ err := c.graphJSON(ctx, http.MethodGet, folderPath+escapeSegment(file.Name)+"?$select=size,folder,createdBy", nil, &item)
if err == nil && item.Folder == nil && item.Size == int64(len(file.Content)) &&
item.CreatedBy.Application != nil && strings.EqualFold(item.CreatedBy.Application.ID, c.clientID) {
return nil
diff --git a/backend/plugin/sharepoint/client_test.go b/backend/plugin/sharepoint/client_test.go
index 7ca6f5a521..d87c91d4ae 100644
--- a/backend/plugin/sharepoint/client_test.go
+++ b/backend/plugin/sharepoint/client_test.go
@@ -91,7 +91,7 @@ func TestResolve(t *testing.T) {
}))
loc, err := client.Resolve(context.Background(), canonical)
require.NoError(t, err)
- require.Equal(t, &Location{DriveID: "d1", ItemID: "root1", WebURL: "https://contoso.sharepoint.com/sites/finance/Shared%20Documents"}, loc)
+ require.Equal(t, &Location{DriveID: "d1", ItemID: "root1", WebURL: "https://contoso.sharepoint.com/sites/finance/Shared%20Documents", url: canonical}, loc)
})
t.Run("folder", func(t *testing.T) {
@@ -100,7 +100,7 @@ func TestResolve(t *testing.T) {
}))
loc, err := client.Resolve(context.Background(), canonical)
require.NoError(t, err)
- require.Equal(t, &Location{DriveID: "d1", ItemID: "item1", WebURL: canonical}, loc)
+ require.Equal(t, &Location{DriveID: "d1", ItemID: "item1", WebURL: canonical, url: canonical}, loc)
})
t.Run("file", func(t *testing.T) {
@@ -138,8 +138,8 @@ type fakeSession struct {
type fakeGraph struct {
t *testing.T
mu sync.Mutex
- // folderIDs maps folder IDs to names.
- folderIDs map[string]string
+ // folders holds the names of the folders under the destination.
+ folders map[string]bool
// files is keyed by "folderName/fileName".
files map[string]fakeFile
sessions map[string]*fakeSession
@@ -153,16 +153,7 @@ type fakeGraph struct {
}
func newFakeGraph(t *testing.T) *fakeGraph {
- return &fakeGraph{t: t, folderIDs: map[string]string{}, files: map[string]fakeFile{}, sessions: map[string]*fakeSession{}}
-}
-
-func (f *fakeGraph) folderID(name string) string {
- for id, folderName := range f.folderIDs {
- if folderName == name {
- return id
- }
- }
- return ""
+ return &fakeGraph{t: t, folders: map[string]bool{}, files: map[string]fakeFile{}, sessions: map[string]*fakeSession{}}
}
func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -177,39 +168,16 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
require.Equal(f.t, "Bearer test-token", r.Header.Get("Authorization"))
- rest, ok := strings.CutPrefix(r.URL.Path, "/drives/d1/items/")
+ // Files are addressed by path under the destination item "dest".
+ rest, ok := strings.CutPrefix(r.URL.Path, "/drives/d1/items/dest:/")
if !ok {
writeGraphError(w, http.StatusNotFound, "itemNotFound")
return
}
- if rest == "dest/children" && r.Method == http.MethodPost {
- var body map[string]any
- require.NoError(f.t, json.NewDecoder(r.Body).Decode(&body))
- require.Equal(f.t, "fail", body["@microsoft.graph.conflictBehavior"])
- name, ok := body["name"].(string)
- require.True(f.t, ok)
- if f.folderID(name) != "" {
- writeGraphError(w, http.StatusConflict, "nameAlreadyExists")
- return
- }
- id := fmt.Sprintf("folder%d", len(f.folderIDs)+1)
- f.folderIDs[id] = name
- writeJSON(w, http.StatusCreated, map[string]any{"id": id, "name": name, "webUrl": "https://sp/" + id, "folder": map[string]any{}})
- return
- }
- itemID, sub, _ := strings.Cut(rest, ":/")
- if itemID == "dest" && r.Method == http.MethodGet {
- id := f.folderID(sub)
- if id == "" {
- writeGraphError(w, http.StatusNotFound, "itemNotFound")
- return
- }
- writeJSON(w, http.StatusOK, map[string]any{"id": id, "webUrl": "https://sp/" + id, "folder": map[string]any{}})
- return
- }
- folderName, ok := f.folderIDs[itemID]
+ folderName, sub, ok := strings.Cut(rest, "/")
if !ok {
- writeGraphError(w, http.StatusNotFound, "itemNotFound")
+ f.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+ writeGraphError(w, http.StatusBadRequest, "invalidRequest")
return
}
switch {
@@ -219,6 +187,7 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeGraphError(w, http.StatusConflict, "nameAlreadyExists")
return
}
+ f.folders[folderName] = true
f.sessionCount++
id := fmt.Sprint(f.sessionCount)
f.sessions[id] = &fakeSession{key: key}
@@ -230,6 +199,7 @@ func (f *fakeGraph) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
data, _ := io.ReadAll(r.Body)
+ f.folders[folderName] = true
f.files[key] = fakeFile{content: data, appID: testAppID}
writeJSON(w, http.StatusCreated, map[string]any{"id": key})
case r.Method == http.MethodGet:
@@ -290,7 +260,7 @@ func (f *fakeGraph) serveUpload(w http.ResponseWriter, r *http.Request) {
const testFolder = "20261022T010203Z [email protected] db 42"
var (
- testLocation = &Location{DriveID: "d1", ItemID: "dest"}
+ testLocation = &Location{DriveID: "d1", ItemID: "dest", WebURL: "https://sp/dest"}
testFiles = []File{
{Name: "statement.sql", Content: []byte("SELECT 1;\n")},
{Name: "result #1.csv", Content: []byte("a,b\n1,2\n")},
@@ -307,12 +277,14 @@ func requireUploaded(t *testing.T, fake *fakeGraph) {
}
}
+const testFolderURL = "https://sp/dest/20261022T010203Z%[email protected]%20db%2042"
+
func TestUpload(t *testing.T) {
fake := newFakeGraph(t)
client, waits := newTestClient(t, fake)
folder, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)
require.NoError(t, err)
- require.Equal(t, &Folder{Name: testFolder, WebURL: "https://sp/folder1"}, folder)
+ require.Equal(t, &Folder{Name: testFolder, WebURL: testFolderURL}, folder)
requireUploaded(t, fake)
require.Equal(t, []string{
"bytes 0-3/10", "bytes 4-7/10", "bytes 8-9/10",
@@ -323,21 +295,11 @@ func TestUpload(t *testing.T) {
// Retrying the same export reuses the folder and the uploaded files.
folder, err = client.Upload(context.Background(), testLocation, testFolder, testFiles)
require.NoError(t, err)
- require.Equal(t, &Folder{Name: testFolder, WebURL: "https://sp/folder1"}, folder)
- require.Len(t, fake.folderIDs, 1)
+ require.Equal(t, &Folder{Name: testFolder, WebURL: testFolderURL}, folder)
+ require.Len(t, fake.folders, 1)
require.Equal(t, 2, fake.sessionCount)
}
-func TestUploadReusesExistingFolder(t *testing.T) {
- fake := newFakeGraph(t)
- fake.folderIDs["existing"] = testFolder
- client, _ := newTestClient(t, fake)
- folder, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)
- require.NoError(t, err)
- require.Equal(t, "https://sp/existing", folder.WebURL)
- requireUploaded(t, fake)
-}
-
func TestUploadFileConflict(t *testing.T) {
files := []File{{Name: "result.csv", Content: []byte("a,b\n")}}
tests := []struct {
@@ -352,7 +314,7 @@ func TestUploadFileConflict(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fake := newFakeGraph(t)
- fake.folderIDs["existing"] = testFolder
+ fake.folders[testFolder] = true
fake.files[testFolder+"/result.csv"] = tc.existing
client, _ := newTestClient(t, fake)
_, err := client.Upload(context.Background(), testLocation, testFolder, files)
@@ -368,27 +330,11 @@ func TestUploadFileConflict(t *testing.T) {
}
}
-func TestUploadFolderCreationFailureUploadsNothing(t *testing.T) {
- fake := newFakeGraph(t)
- fake.override = func(w http.ResponseWriter, r *http.Request) bool {
- if r.URL.Path != "/drives/d1/items/dest/children" {
- return false
- }
- writeGraphError(w, http.StatusForbidden, "accessDenied")
- return true
- }
- client, _ := newTestClient(t, fake)
- _, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)
- require.ErrorIs(t, err, ErrNothingUploaded)
- require.ErrorIs(t, err, ErrNotFoundOrNoAccess)
- require.Zero(t, fake.sessionCount)
-}
-
func TestUploadHonorsRetryAfter(t *testing.T) {
fake := newFakeGraph(t)
throttled := false
- fake.override = func(w http.ResponseWriter, r *http.Request) bool {
- if throttled || r.URL.Path != "/drives/d1/items/dest/children" {
+ fake.override = func(w http.ResponseWriter, _ *http.Request) bool {
+ if throttled {
return false
}
throttled = true
@@ -435,12 +381,138 @@ func TestUploadUnconfirmed(t *testing.T) {
require.Equal(t, []time.Duration{time.Second, 2 * time.Second}, *waits)
}
-func TestUploadForbidden(t *testing.T) {
- client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+func TestUploadForbiddenUploadsNothing(t *testing.T) {
+ fake := newFakeGraph(t)
+ fake.override = func(w http.ResponseWriter, _ *http.Request) bool {
writeGraphError(w, http.StatusForbidden, "accessDenied")
- }))
+ return true
+ }
+ client, _ := newTestClient(t, fake)
+ client.maxConcurrentUploads = len(testFiles)
_, err := client.Upload(context.Background(), testLocation, testFolder, testFiles)
require.ErrorIs(t, err, ErrNotFoundOrNoAccess)
+ require.ErrorIs(t, err, ErrNothingUploaded)
+ require.Empty(t, fake.folders)
+}
+
+func TestUploadRejectionDoesNotDenyAConcurrentUpload(t *testing.T) {
+ files := []File{{Name: "a.sql", Content: []byte("a")}, {Name: "b.csv", Content: []byte("b")}}
+ rejected := make(chan struct{})
+ client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.Path, "b.csv") {
+ writeGraphError(w, http.StatusForbidden, "accessDenied")
+ close(rejected)
+ return
+ }
+ <-rejected
+ writeJSON(w, http.StatusCreated, map[string]any{"id": "a"})
+ }))
+ client.simpleUploadMaxSize = simpleUploadMaxSize
+ client.maxConcurrentUploads = len(files)
+ _, err := client.Upload(context.Background(), testLocation, testFolder, files)
+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)
+ require.NotErrorIs(t, err, ErrNothingUploaded)
+}
+
+// recreatableDestination serves a destination URL that resolves to the item
+// named by current, under drive d1. Only the item "dest" holds files.
+type recreatableDestination struct {
+ fake *fakeGraph
+ current string
+ resolve int
+}
+
+func newRecreatableDestination(t *testing.T, current string) *recreatableDestination {
+ d := &recreatableDestination{fake: newFakeGraph(t), current: current}
+ d.fake.override = func(w http.ResponseWriter, r *http.Request) bool {
+ if !strings.HasPrefix(r.URL.Path, "/shares/") {
+ return false
+ }
+ d.resolve++
+ if d.current == "" {
+ writeGraphError(w, http.StatusForbidden, "accessDenied")
+ return true
+ }
+ writeJSON(w, http.StatusOK, map[string]any{
+ "id": d.current, "webUrl": "https://sp/" + d.current, "folder": map[string]any{},
+ "parentReference": map[string]any{"driveId": "d1"},
+ })
+ return true
+ }
+ return d
+}
+
+func TestResolveRecent(t *testing.T) {
+ dest := newRecreatableDestination(t, "dest")
+ client, _ := newTestClient(t, dest.fake)
+ now := time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)
+ client.now = func() time.Time { return now }
+ ctx := context.Background()
+
+ loc, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, "dest", loc.ItemID)
+ _, err = client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, 1, dest.resolve)
+
+ // Resolve always asks Graph.
+ _, err = client.Resolve(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, 2, dest.resolve)
+
+ now = now.Add(recentResolveTTL)
+ _, err = client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, 3, dest.resolve)
+}
+
+func TestUploadResolvesReusedDestinationAgain(t *testing.T) {
+ ctx := context.Background()
+ t.Run("recreated destination receives the export", func(t *testing.T) {
+ dest := newRecreatableDestination(t, "deleted")
+ client, _ := newTestClient(t, dest.fake)
+ _, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ dest.current = "dest"
+ loc, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, "deleted", loc.ItemID)
+
+ folder, err := client.Upload(ctx, loc, testFolder, testFiles)
+ require.NoError(t, err)
+ require.Equal(t, testFolderURL, folder.WebURL)
+ requireUploaded(t, dest.fake)
+ require.Equal(t, 2, dest.resolve)
+ })
+ t.Run("deleted destination fails without uploading", func(t *testing.T) {
+ dest := newRecreatableDestination(t, "deleted")
+ client, _ := newTestClient(t, dest.fake)
+ _, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ dest.current = ""
+ loc, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+
+ _, err = client.Upload(ctx, loc, testFolder, testFiles)
+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)
+ require.ErrorIs(t, err, ErrNothingUploaded)
+ require.Equal(t, 2, dest.resolve)
+ })
+ t.Run("fresh resolution is not resolved again", func(t *testing.T) {
+ dest := newRecreatableDestination(t, "deleted")
+ client, _ := newTestClient(t, dest.fake)
+ loc, err := client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+
+ _, err = client.Upload(ctx, loc, testFolder, testFiles)
+ require.ErrorIs(t, err, ErrNotFoundOrNoAccess)
+ require.Equal(t, 1, dest.resolve)
+ // The failure drops the resolution, so the next export asks Graph.
+ _, err = client.ResolveRecent(ctx, resolveShareURL)
+ require.NoError(t, err)
+ require.Equal(t, 2, dest.resolve)
+ })
}
func TestTransportErrorOmitsURL(t *testing.T) {
@@ -460,8 +532,8 @@ func TestUploadSmallFilesInOneRequestEach(t *testing.T) {
require.NoError(t, err)
requireUploaded(t, fake)
require.Zero(t, fake.sessionCount)
- // One request creates the folder and one uploads each file.
- require.Equal(t, 1+len(testFiles), fake.requestCount)
+ // One request uploads each file; the first also creates the folder.
+ require.Equal(t, len(testFiles), fake.requestCount)
}
func TestUploadInParallel(t *testing.T) {
@@ -470,10 +542,6 @@ func TestUploadInParallel(t *testing.T) {
inFlight := 0
allInFlight := make(chan struct{})
client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Method == http.MethodPost {
- writeJSON(w, http.StatusCreated, map[string]any{"id": "folder1", "webUrl": "https://sp/folder1", "folder": map[string]any{}})
- return
- }
mu.Lock()
inFlight++
if inFlight == len(files) {
@@ -492,3 +560,25 @@ func TestUploadInParallel(t *testing.T) {
_, err := client.Upload(context.Background(), testLocation, testFolder, files)
require.NoError(t, err)
}
+
+func TestUploadRejectedAfterLostResponseIsUnconfirmed(t *testing.T) {
+ puts := 0
+ client, _ := newTestClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodPut {
+ puts++
+ if puts == 1 {
+ // The first attempt is committed but its response is lost.
+ conn, _, err := w.(http.Hijacker).Hijack()
+ require.NoError(t, err)
+ _ = conn.Close()
+ return
+ }
+ }
+ writeGraphError(w, http.StatusForbidden, "accessDenied")
+ }))
+ client.simpleUploadMaxSize = simpleUploadMaxSize
+ _, err := client.Upload(context.Background(), testLocation, testFolder, []File{{Name: "a.sql", Content: []byte("a")}})
+ require.ErrorIs(t, err, ErrUnconfirmed)
+ require.NotErrorIs(t, err, ErrNothingUploaded)
+ require.Equal(t, 2, puts)
+}
diff --git a/backend/plugin/sharepoint/sharepoint.go b/backend/plugin/sharepoint/sharepoint.go
index c239acbd2c..a63caee420 100644
--- a/backend/plugin/sharepoint/sharepoint.go
+++ b/backend/plugin/sharepoint/sharepoint.go
@@ -21,6 +21,11 @@ type Location struct {
DriveID string
ItemID string
WebURL string
+
+ // url is the canonical URL the location was resolved from.
+ url string
+ // reused reports whether ResolveRecent returned an earlier resolution.
+ reused bool
}
// File is one file of an export.
@@ -60,9 +65,14 @@ type Client interface {
// Resolve resolves a canonical URL from NormalizeURL to a library root or
// folder. Resolving proves only that the destination is readable.
Resolve(ctx context.Context, canonicalURL string) (*Location, error)
+ // ResolveRecent is Resolve, except that it reuses a resolution of the same
+ // URL made within the last minute.
+ ResolveRecent(ctx context.Context, canonicalURL string) (*Location, error)
// Upload creates folderName under loc and uploads files into it. Retrying
// Upload with the same arguments reuses the folder and files already
- // uploaded by the application instead of creating copies.
+ // uploaded by the application instead of creating copies. If loc is a
+ // reused resolution that Graph no longer finds, Upload resolves its URL
+ // again and retries once.
Upload(ctx context.Context, loc *Location, folderName string, files []File) (*Folder, error)
}
diff --git a/backend/plugin/sharepoint/transport.go b/backend/plugin/sharepoint/transport.go
index 5402d15949..e2cefa1d86 100644
--- a/backend/plugin/sharepoint/transport.go
+++ b/backend/plugin/sharepoint/transport.go
@@ -24,6 +24,9 @@ type response struct {
status int
header http.Header
body []byte
+ // afterUnknownOutcome reports that an earlier attempt of the request may
+ // have taken effect.
+ afterUnknownOutcome bool
}
// unknownOutcomeError means a request may or may not have taken effect.
@@ -43,6 +46,9 @@ type graphError struct {
status int
code string
message string
+ // afterUnknownOutcome reports that an earlier attempt of the request may
+ // have taken effect, so the error does not prove the request did nothing.
+ afterUnknownOutcome bool
}
func newGraphError(resp *response) *graphError {
@@ -53,7 +59,7 @@ func newGraphError(resp *response) *graphError {
} `json:"error"`
}
_ = json.Unmarshal(resp.body, &body)
- return &graphError{status: resp.status, code: body.Error.Code, message: body.Error.Message}
+ return &graphError{status: resp.status, code: body.Error.Code, message: body.Error.Message, afterUnknownOutcome: resp.afterUnknownOutcome}
}
func (e *graphError) sentinel() error {
@@ -161,6 +167,7 @@ func (c *graphClient) send(ctx context.Context, method, target string, body []by
outcomeUnknown = true
lastErr = newGraphError(resp)
default:
+ resp.afterUnknownOutcome = outcomeUnknown
return resp, nil
}
if attempt >= c.maxAttempts {
diff --git a/backend/tests/sql_export_masking_test.go b/backend/tests/sql_export_masking_test.go
new file mode 100644
index 0000000000..e7956f1d4d
--- /dev/null
+++ b/backend/tests/sql_export_masking_test.go
@@ -0,0 +1,120 @@
+package tests
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/alexmullins/zip"
+ "github.com/stretchr/testify/require"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// TestSQLExportMatchesQueryMasking checks that an export masks a column the
+// same way a query does. MD5 masking is not idempotent, so masking a result
+// twice changes the exported value.
+func TestSQLExportMatchesQueryMasking(t *testing.T) {
+ t.Parallel()
+ a := require.New(t)
+ ctl, ctx := startWorkspace(context.Background(), t)
+
+ _, err := ctl.settingServiceClient.UpdateSetting(ctx, connect.NewRequest(&v1pb.UpdateSettingRequest{
+ AllowMissing: true,
+ Setting: &v1pb.Setting{
+ Name: "settings/" + v1pb.Setting_SEMANTIC_TYPES.String(),
+ Value: &v1pb.SettingValue{
+ Value: &v1pb.SettingValue_SemanticType{
+ SemanticType: &v1pb.SemanticTypeSetting{
+ Types: []*v1pb.SemanticTypeSetting_SemanticType{{
+ Id: "hashed",
+ Title: "Hashed",
+ Algorithm: &v1pb.Algorithm{
+ Mask: &v1pb.Algorithm_Md5Mask{Md5Mask: &v1pb.Algorithm_MD5Mask{Salt: "salt"}},
+ },
+ }},
+ },
+ },
+ },
+ },
+ }))
+ a.NoError(err)
+
+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("export-masking"),
+ Instance: &v1pb.Instance{
+ SyncDatabases: &v1pb.SyncDatabases{},
+ Title: "Export masking",
+ Engine: v1pb.Engine_POSTGRES,
+ Environment: new("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{sharedPgTarget(t).adminDataSource()},
+ },
+ }))
+ a.NoError(err)
+ databaseName := generateRandomString("maskdb")
+ a.NoError(ctl.createDatabase(ctx, ctl.project, instanceResp.Msg, nil, databaseName, ""))
+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
+ Name: fmt.Sprintf("%s/databases/%s", instanceResp.Msg.Name, databaseName),
+ }))
+ a.NoError(err)
+ database := databaseResp.Msg
+
+ setup, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: ctl.project.Name,
+ Sheet: &v1pb.Sheet{Content: []byte("CREATE TABLE employee(id INT PRIMARY KEY, secret TEXT); INSERT INTO employee VALUES (1, 'plain');")},
+ }))
+ a.NoError(err)
+ a.NoError(ctl.changeDatabase(ctx, ctl.project, database, setup.Msg, false))
+ _, err = ctl.databaseCatalogServiceClient.UpdateDatabaseCatalog(ctx, connect.NewRequest(&v1pb.UpdateDatabaseCatalogRequest{
+ Catalog: &v1pb.DatabaseCatalog{
+ Name: database.Name + "/catalog",
+ Schemas: []*v1pb.SchemaCatalog{{
+ Name: "public",
+ Tables: []*v1pb.TableCatalog{{
+ Name: "employee",
+ Kind: &v1pb.TableCatalog_Columns_{Columns: &v1pb.TableCatalog_Columns{
+ Columns: []*v1pb.ColumnCatalog{{Name: "secret", SemanticType: "hashed"}},
+ }},
+ }},
+ }},
+ },
+ }))
+ a.NoError(err)
+
+ const statement = "SELECT secret FROM employee"
+ queryResp, err := ctl.sqlServiceClient.Query(ctx, connect.NewRequest(&v1pb.QueryRequest{Name: database.Name, Statement: statement}))
+ a.NoError(err)
+ a.Len(queryResp.Msg.Results, 1)
+ a.Empty(queryResp.Msg.Results[0].Error)
+ a.Len(queryResp.Msg.Results[0].Rows, 1)
+ queried := queryResp.Msg.Results[0].Rows[0].Values[0].GetStringValue()
+ a.NotEqual("plain", queried, "precondition: the semantic type must mask the column")
+
+ exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(&v1pb.ExportRequest{
+ Name: database.Name,
+ Format: v1pb.ExportFormat_CSV,
+ Statement: statement,
+ }))
+ a.NoError(err)
+ content := exportResp.Msg.Content
+ zipReader, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
+ a.NoError(err)
+ var exported string
+ for _, file := range zipReader.File {
+ if !strings.HasSuffix(file.Name, ".result.csv") {
+ continue
+ }
+ r, err := file.Open()
+ a.NoError(err)
+ data, err := io.ReadAll(r)
+ a.NoError(err)
+ exported = string(data)
+ }
+ // CSV export quotes text values.
+ a.Equal("secret\n\""+queried+"\"", exported)
+}
diff --git a/docs/adr/0007-sharepoint-export-ownership-boundary.md b/docs/adr/0007-sharepoint-export-ownership-boundary.md
index e0c2ddcff8..bca1309b23 100644
--- a/docs/adr/0007-sharepoint-export-ownership-boundary.md
+++ b/docs/adr/0007-sharepoint-export-ownership-boundary.md
@@ -111,9 +111,10 @@ the same export file names. If delivery remains uncertain, Bytebase reports that
delivery could not be confirmed and files may exist in SharePoint. It does
not claim that no file was created.
-Bytebase checks the destination once, before the query runs. A change made
-while an export is running applies to the next export. A rejected destination
-is never redirected to another one.
+Bytebase checks the destination once, before the query runs, reusing a check of
+the same URL made within the last minute so a batch export resolves its
+destination once. A change made while an export is running applies to the next
+export. A rejected destination is never redirected to another one.
Internal upload retries reuse the generated files, file names, and captured
destination without rerunning the query. A new Export action starts a new export
@@ -138,9 +139,13 @@ URL that does not resolve. It stores and returns only the canonical URL; resolve
names go stale on rename and are not kept. A SharePoint browsing picker is
outside the initial scope.
-Bytebase resolves the stored URL at each export rather than pinning item IDs.
-Renaming or moving the folder makes exports fail as not found; a folder
-recreated at the same URL receives subsequent exports.
+Bytebase resolves the stored URL rather than pinning item IDs, and reuses a
+resolution for at most a minute. Renaming or moving the folder makes exports
+fail as not found, except that an export reusing a resolution from before the
+rename uploads into the renamed folder and returns a link to the old path. When
+SharePoint rejects an upload into a reused resolution as not found, Bytebase
+resolves the URL again and retries once, so a folder recreated at the same URL
+receives that export.
Customers create and rotate Entra client secrets. Expired or invalid credentials
cause affected exports to fail with a clear credential error. An authorized user