Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

pikpak: fix downloads that return an HTTP error page as the file - fixes #10187

Cestercianc9f491800b7d
diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.go
index a765efa06..ac462cd7b 100644
--- a/backend/pikpak/pikpak.go
+++ b/backend/pikpak/pikpak.go
@@ -1946,6 +1946,12 @@ func (o *Object) httpResponse(ctx context.Context, url, method string, options [
}
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req)
+ // client.Do does not fail a completed non-2xx response, so the
+ // HTML error page would be returned as the file. errorHandler
+ // closes the body; shouldRetry then retries retryErrorCodes.
+ if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
+ err = errorHandler(res)
+ }
return o.fs.shouldRetry(ctx, res, err)
})
if err != nil {
diff --git a/backend/pikpak/pikpak_internal_test.go b/backend/pikpak/pikpak_internal_test.go
new file mode 100644
index 000000000..ee1571a12
--- /dev/null
+++ b/backend/pikpak/pikpak_internal_test.go
@@ -0,0 +1,185 @@
+package pikpak
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "sync/atomic"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// closeTracker counts Close calls on a response body.
+type closeTracker struct {
+ io.ReadCloser
+ closed *atomic.Int32
+}
+
+func (b closeTracker) Close() error {
+ b.closed.Add(1)
+ return b.ReadCloser.Close()
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+ return f(r)
+}
+
+// newDownloadTestObject returns an object whose downloads are served by srv.
+// closed counts response bodies closed through the client.
+func newDownloadTestObject(t *testing.T, srv *httptest.Server, closed *atomic.Int32) *Object {
+ t.Helper()
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.LowLevelRetries = 3
+ client := *srv.Client()
+ base := client.Transport
+ if base == nil {
+ base = http.DefaultTransport
+ }
+ client.Transport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ res, err := base.RoundTrip(req)
+ if err == nil && res != nil && res.Body != nil {
+ res.Body = closeTracker{ReadCloser: res.Body, closed: closed}
+ }
+ return res, err
+ })
+ return &Object{
+ fs: &Fs{
+ client: &client,
+ pacer: fs.NewPacer(ctx, &pacer.ZeroDelayCalculator{}),
+ },
+ size: 1,
+ }
+}
+
+// TestOpenHTTPStatus checks that a non-2xx download response is an error
+// (body closed, retriable statuses retried by the pacer) and that 200 and
+// 206 responses are still returned as file content.
+func TestOpenHTTPStatus(t *testing.T) {
+ const htmlBody = "<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"
+ fileBody := []byte("pikpak-file-bytes-0123456789")
+
+ t.Run("502", func(t *testing.T) {
+ var closed atomic.Int32
+ var hits atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ w.Header().Set("Content-Type", "text/html")
+ w.WriteHeader(http.StatusBadGateway)
+ _, _ = io.WriteString(w, htmlBody)
+ }))
+ t.Cleanup(srv.Close)
+ o := newDownloadTestObject(t, srv, &closed)
+ in, err := o.open(context.Background(), srv.URL)
+ if err == nil {
+ got, readErr := io.ReadAll(in)
+ require.NoError(t, readErr)
+ require.NoError(t, in.Close())
+ t.Fatalf("open returned the error page as file data (%d bytes): %q", len(got), got)
+ }
+ assert.Nil(t, in)
+ assert.ErrorContains(t, err, "502")
+ assert.NotContains(t, err.Error(), htmlBody)
+ assert.Equal(t, int32(3), hits.Load(), "502 should be retried by the pacer")
+ assert.Equal(t, hits.Load(), closed.Load(), "error response bodies should be closed")
+ })
+
+ t.Run("404", func(t *testing.T) {
+ var closed atomic.Int32
+ var hits atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ http.Error(w, "missing", http.StatusNotFound)
+ }))
+ t.Cleanup(srv.Close)
+ o := newDownloadTestObject(t, srv, &closed)
+ in, err := o.open(context.Background(), srv.URL)
+ require.Error(t, err)
+ assert.Nil(t, in)
+ assert.Equal(t, int32(1), hits.Load(), "404 is not a retry status")
+ assert.Equal(t, hits.Load(), closed.Load())
+ })
+
+ t.Run("502 then 200", func(t *testing.T) {
+ var closed atomic.Int32
+ var hits atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if hits.Add(1) == 1 {
+ w.Header().Set("Content-Type", "text/html")
+ w.WriteHeader(http.StatusBadGateway)
+ _, _ = io.WriteString(w, htmlBody)
+ return
+ }
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(fileBody)
+ }))
+ t.Cleanup(srv.Close)
+ o := newDownloadTestObject(t, srv, &closed)
+ in, err := o.open(context.Background(), srv.URL)
+ require.NoError(t, err)
+ got, err := io.ReadAll(in)
+ require.NoError(t, err)
+ require.NoError(t, in.Close())
+ assert.Equal(t, fileBody, got)
+ assert.Equal(t, int32(2), hits.Load())
+ assert.Equal(t, int32(2), closed.Load())
+ })
+
+ t.Run("200", func(t *testing.T) {
+ var closed atomic.Int32
+ var hits atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ if r.Header.Get("Range") != "" {
+ t.Errorf("unexpected Range header %q", r.Header.Get("Range"))
+ }
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(fileBody)
+ }))
+ t.Cleanup(srv.Close)
+ o := newDownloadTestObject(t, srv, &closed)
+ o.size = int64(len(fileBody))
+ in, err := o.open(context.Background(), srv.URL)
+ require.NoError(t, err)
+ got, err := io.ReadAll(in)
+ require.NoError(t, err)
+ assert.Equal(t, int32(0), closed.Load(), "successful body stays open for the caller")
+ require.NoError(t, in.Close())
+ assert.Equal(t, fileBody, got)
+ assert.Equal(t, int32(1), hits.Load())
+ assert.Equal(t, int32(1), closed.Load())
+ })
+
+ t.Run("206", func(t *testing.T) {
+ var closed atomic.Int32
+ var hits atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ assert.Equal(t, "bytes=0-3", r.Header.Get("Range"))
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Range", "bytes 0-3/"+strconv.Itoa(len(fileBody)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(fileBody[:4])
+ }))
+ t.Cleanup(srv.Close)
+ o := newDownloadTestObject(t, srv, &closed)
+ o.size = int64(len(fileBody))
+ in, err := o.open(context.Background(), srv.URL, &fs.RangeOption{Start: 0, End: 3})
+ require.NoError(t, err)
+ got, err := io.ReadAll(in)
+ require.NoError(t, err)
+ require.NoError(t, in.Close())
+ assert.Equal(t, fileBody[:4], got)
+ assert.Equal(t, int32(1), hits.Load())
+ assert.Equal(t, int32(1), closed.Load())
+ })
+}