Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

drive: backend query command

John-Paul Smithca903b987279
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 899c7daf9..a62d47ca9 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -3555,6 +3555,50 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
return nil
}
+func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
+ list := f.svc.Files.List()
+ if query != "" {
+ list.Q(query)
+ }
+
+ if f.opt.ListChunk > 0 {
+ list.PageSize(f.opt.ListChunk)
+ }
+ list.SupportsAllDrives(true)
+ list.IncludeItemsFromAllDrives(true)
+ if f.isTeamDrive && !f.opt.SharedWithMe {
+ list.DriveId(f.opt.TeamDriveID)
+ list.Corpora("drive")
+ }
+ // If using appDataFolder then need to add Spaces
+ if f.rootFolderID == "appDataFolder" {
+ list.Spaces("appDataFolder")
+ }
+
+ fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
+
+ var results []*drive.File
+ for {
+ var files *drive.FileList
+ err = f.pacer.Call(func() (bool, error) {
+ files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
+ return f.shouldRetry(ctx, err)
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to execute query: %w", err)
+ }
+ if files.IncompleteSearch {
+ fs.Errorf(f, "search result INCOMPLETE")
+ }
+ results = append(results, files.Files...)
+ if files.NextPageToken == "" {
+ break
+ }
+ list.PageToken(files.NextPageToken)
+ }
+ return results, nil
+}
+
var commandHelp = []fs.CommandHelp{{
Name: "get",
Short: "Get command for fetching the drive config parameters",
@@ -3705,6 +3749,47 @@ Use the --interactive/-i or --dry-run flag to see what would be copied before co
}, {
Name: "importformats",
Short: "Dump the import formats for debug purposes",
+}, {
+ Name: "query",
+ Short: "List files using Google Drive query language",
+ Long: `This command lists files based on a query
+
+Usage:
+
+ rclone backend query drive: query
+
+The query syntax is documented at [Google Drive Search query terms and
+operators](https://developers.google.com/drive/api/guides/ref-search-terms).
+
+For example:
+
+ rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
+
+If the query contains literal ' or \ characters, these need to be escaped with
+\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a
+file named "foo ' \.txt":
+
+ rclone backend query drive: "name = 'foo \' \\\.txt'"
+
+The result is a JSON array of matches, for example:
+
+[
+ {
+ "createdTime": "2017-06-29T19:58:28.537Z",
+ "id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
+ "md5Checksum": "68518d16be0c6fbfab918be61d658032",
+ "mimeType": "text/plain",
+ "modifiedTime": "2024-02-02T10:40:02.874Z",
+ "name": "foo ' \\.txt",
+ "parents": [
+ "0BxAe_BCDE4zkFGZpcWJGek0xbzC"
+ ],
+ "resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
+ "sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
+ "size": "311",
+ "webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
+ }
+]`,
}}
// Command the backend to run a named command
@@ -3822,6 +3907,17 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
return f.exportFormats(ctx), nil
case "importformats":
return f.importFormats(ctx), nil
+ case "query":
+ if len(arg) == 1 {
+ query := arg[0]
+ var results, err = f.query(ctx, query)
+ if err != nil {
+ return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
+ }
+ return results, nil
+ } else {
+ return nil, errors.New("need a query argument")
+ }
default:
return nil, fs.ErrorCommandNotFound
}
diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go
index c2e6da05b..838962759 100644
--- a/backend/drive/drive_internal_test.go
+++ b/backend/drive/drive_internal_test.go
@@ -524,6 +524,41 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
})
}
+// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
+func (f *Fs) InternalTestQuery(t *testing.T) {
+ ctx := context.Background()
+ var err error
+ t.Run("BadQuery", func(t *testing.T) {
+ _, err = f.query(ctx, "this is a bad query")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "failed to execute query")
+ })
+
+ t.Run("NoMatch", func(t *testing.T) {
+ results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
+ require.NoError(t, err)
+ assert.Len(t, results, 0)
+ })
+
+ t.Run("GoodQuery", func(t *testing.T) {
+ pathSegments := strings.Split(existingFile, "/")
+ var parent string
+ for _, item := range pathSegments {
+ // the file name contains ' characters which must be escaped
+ escapedItem := f.opt.Enc.FromStandardName(item)
+ escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
+ escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
+
+ results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ assert.Len(t, results[0].Id, 33)
+ assert.Equal(t, results[0].Name, item)
+ parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
+ }
+ })
+}
+
// TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
func (f *Fs) InternalTestAgeQuery(t *testing.T) {
// Check set up for filtering
@@ -611,6 +646,7 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("Shortcuts", f.InternalTestShortcuts)
t.Run("UnTrash", f.InternalTestUnTrash)
t.Run("CopyID", f.InternalTestCopyID)
+ t.Run("Query", f.InternalTestQuery)
t.Run("AgeQuery", f.InternalTestAgeQuery)
t.Run("ShouldRetry", f.InternalTestShouldRetry)
}