"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: fix downloads that return an HTTP error page as the file - fixes #10187
Cestercianc9f491800b7d
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex a765efa06..ac462cd7b 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -1946,6 +1946,12 @@ func (o *Object) httpResponse(ctx context.Context, url, method string, options [}err = o.fs.pacer.Call(func() (bool, error) {res, err = o.fs.client.Do(req)+ // client.Do does not fail a completed non-2xx response, so the+ // HTML error page would be returned as the file. errorHandler+ // closes the body; shouldRetry then retries retryErrorCodes.+ if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {+ err = errorHandler(res)+ }return o.fs.shouldRetry(ctx, res, err)})if err != nil {diff --git a/backend/pikpak/pikpak_internal_test.go b/backend/pikpak/pikpak_internal_test.gonew file mode 100644index 000000000..ee1571a12--- /dev/null+++ b/backend/pikpak/pikpak_internal_test.go@@ -0,0 +1,185 @@+package pikpak++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "strconv"+ "sync/atomic"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// closeTracker counts Close calls on a response body.+type closeTracker struct {+ io.ReadCloser+ closed *atomic.Int32+}++func (b closeTracker) Close() error {+ b.closed.Add(1)+ return b.ReadCloser.Close()+}++type roundTripperFunc func(*http.Request) (*http.Response, error)++func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {+ return f(r)+}++// newDownloadTestObject returns an object whose downloads are served by srv.+// closed counts response bodies closed through the client.+func newDownloadTestObject(t *testing.T, srv *httptest.Server, closed *atomic.Int32) *Object {+ t.Helper()+ ctx, ci := fs.AddConfig(context.Background())+ ci.LowLevelRetries = 3+ client := *srv.Client()+ base := client.Transport+ if base == nil {+ base = http.DefaultTransport+ }+ client.Transport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {+ res, err := base.RoundTrip(req)+ if err == nil && res != nil && res.Body != nil {+ res.Body = closeTracker{ReadCloser: res.Body, closed: closed}+ }+ return res, err+ })+ return &Object{+ fs: &Fs{+ client: &client,+ pacer: fs.NewPacer(ctx, &pacer.ZeroDelayCalculator{}),+ },+ size: 1,+ }+}++// TestOpenHTTPStatus checks that a non-2xx download response is an error+// (body closed, retriable statuses retried by the pacer) and that 200 and+// 206 responses are still returned as file content.+func TestOpenHTTPStatus(t *testing.T) {+ const htmlBody = "<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"+ fileBody := []byte("pikpak-file-bytes-0123456789")++ t.Run("502", func(t *testing.T) {+ var closed atomic.Int32+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ w.Header().Set("Content-Type", "text/html")+ w.WriteHeader(http.StatusBadGateway)+ _, _ = io.WriteString(w, htmlBody)+ }))+ t.Cleanup(srv.Close)+ o := newDownloadTestObject(t, srv, &closed)+ in, err := o.open(context.Background(), srv.URL)+ if err == nil {+ got, readErr := io.ReadAll(in)+ require.NoError(t, readErr)+ require.NoError(t, in.Close())+ t.Fatalf("open returned the error page as file data (%d bytes): %q", len(got), got)+ }+ assert.Nil(t, in)+ assert.ErrorContains(t, err, "502")+ assert.NotContains(t, err.Error(), htmlBody)+ assert.Equal(t, int32(3), hits.Load(), "502 should be retried by the pacer")+ assert.Equal(t, hits.Load(), closed.Load(), "error response bodies should be closed")+ })++ t.Run("404", func(t *testing.T) {+ var closed atomic.Int32+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ http.Error(w, "missing", http.StatusNotFound)+ }))+ t.Cleanup(srv.Close)+ o := newDownloadTestObject(t, srv, &closed)+ in, err := o.open(context.Background(), srv.URL)+ require.Error(t, err)+ assert.Nil(t, in)+ assert.Equal(t, int32(1), hits.Load(), "404 is not a retry status")+ assert.Equal(t, hits.Load(), closed.Load())+ })++ t.Run("502 then 200", func(t *testing.T) {+ var closed atomic.Int32+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if hits.Add(1) == 1 {+ w.Header().Set("Content-Type", "text/html")+ w.WriteHeader(http.StatusBadGateway)+ _, _ = io.WriteString(w, htmlBody)+ return+ }+ w.Header().Set("Content-Type", "application/octet-stream")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write(fileBody)+ }))+ t.Cleanup(srv.Close)+ o := newDownloadTestObject(t, srv, &closed)+ in, err := o.open(context.Background(), srv.URL)+ require.NoError(t, err)+ got, err := io.ReadAll(in)+ require.NoError(t, err)+ require.NoError(t, in.Close())+ assert.Equal(t, fileBody, got)+ assert.Equal(t, int32(2), hits.Load())+ assert.Equal(t, int32(2), closed.Load())+ })++ t.Run("200", func(t *testing.T) {+ var closed atomic.Int32+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ if r.Header.Get("Range") != "" {+ t.Errorf("unexpected Range header %q", r.Header.Get("Range"))+ }+ w.Header().Set("Content-Type", "application/octet-stream")+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write(fileBody)+ }))+ t.Cleanup(srv.Close)+ o := newDownloadTestObject(t, srv, &closed)+ o.size = int64(len(fileBody))+ in, err := o.open(context.Background(), srv.URL)+ require.NoError(t, err)+ got, err := io.ReadAll(in)+ require.NoError(t, err)+ assert.Equal(t, int32(0), closed.Load(), "successful body stays open for the caller")+ require.NoError(t, in.Close())+ assert.Equal(t, fileBody, got)+ assert.Equal(t, int32(1), hits.Load())+ assert.Equal(t, int32(1), closed.Load())+ })++ t.Run("206", func(t *testing.T) {+ var closed atomic.Int32+ var hits atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ assert.Equal(t, "bytes=0-3", r.Header.Get("Range"))+ w.Header().Set("Content-Type", "application/octet-stream")+ w.Header().Set("Content-Range", "bytes 0-3/"+strconv.Itoa(len(fileBody)))+ w.WriteHeader(http.StatusPartialContent)+ _, _ = w.Write(fileBody[:4])+ }))+ t.Cleanup(srv.Close)+ o := newDownloadTestObject(t, srv, &closed)+ o.size = int64(len(fileBody))+ in, err := o.open(context.Background(), srv.URL, &fs.RangeOption{Start: 0, End: 3})+ require.NoError(t, err)+ got, err := io.ReadAll(in)+ require.NoError(t, err)+ require.NoError(t, in.Close())+ assert.Equal(t, fileBody[:4], got)+ assert.Equal(t, int32(1), hits.Load())+ assert.Equal(t, int32(1), closed.Load())+ })+}