"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pool: Add ability to wait for a write to RW
Nick Craig-Woodd08b49d723b7
diff --git a/lib/pool/reader_writer.go b/lib/pool/reader_writer.goindex c6cf2caea..f7c263bd1 100644--- a/lib/pool/reader_writer.go+++ b/lib/pool/reader_writer.go@@ -1,9 +1,11 @@package poolimport (+ "context""errors""io""sync"+ "time")// RWAccount is a function which will be called after every read@@ -24,10 +26,11 @@ type RW struct {// Shared variables between Read and Write// Write updates these but Read reads from them// They must all stay in sync together- mu sync.Mutex // protect the shared variables- pages [][]byte // backing store- size int // size written- lastOffset int // size in last page+ mu sync.Mutex // protect the shared variables+ pages [][]byte // backing store+ size int // size written+ lastOffset int // size in last page+ written chan struct{} // signalled when a write happens// Read side Variablesout int // offset we are reading from@@ -48,10 +51,12 @@ var (//// When writing it only appends data. Seek only applies to reading.func NewRW(pool *Pool) *RW {- return &RW{- pool: pool,- pages: make([][]byte, 0, 16),+ rw := &RW{+ pool: pool,+ pages: make([][]byte, 0, 16),+ written: make(chan struct{}, 1),}+ return rw}// SetAccounting should be provided with a function which will be@@ -217,6 +222,7 @@ func (rw *RW) Write(p []byte) (n int, err error) {rw.size += nnrw.lastOffset += nnrw.mu.Unlock()+ rw.signalWrite() // signal more data available}return n, nil}@@ -240,6 +246,7 @@ func (rw *RW) ReadFrom(r io.Reader) (n int64, err error) {rw.size += nnrw.lastOffset += nnrw.mu.Unlock()+ rw.signalWrite() // signal more data available}if err == io.EOF {err = nil@@ -247,6 +254,29 @@ func (rw *RW) ReadFrom(r io.Reader) (n int64, err error) {return n, err}+// signal that a write has happened+func (rw *RW) signalWrite() {+ select {+ case rw.written <- struct{}{}:+ default:+ }+}++// WaitWrite sleeps until a data is written to the RW or Close is+// called or the context is cancelled occurs or for a maximum of 1+// Second then returns.+//+// This can be used when calling Read while the buffer is filling up.+func (rw *RW) WaitWrite(ctx context.Context) {+ timer := time.NewTimer(time.Second)+ select {+ case <-timer.C:+ case <-ctx.Done():+ case <-rw.written:+ }+ timer.Stop()+}+// Seek sets the offset for the next Read (not Write - this is always// appended) to offset, interpreted according to whence: SeekStart// means relative to the start of the file, SeekCurrent means relative@@ -286,6 +316,7 @@ func (rw *RW) Seek(offset int64, whence int) (int64, error) {func (rw *RW) Close() error {rw.mu.Lock()defer rw.mu.Unlock()+ rw.signalWrite() // signal more data availablefor _, page := range rw.pages {rw.pool.Put(page)}diff --git a/lib/pool/reader_writer_test.go b/lib/pool/reader_writer_test.goindex 7f5315fc6..d51c70b58 100644--- a/lib/pool/reader_writer_test.go+++ b/lib/pool/reader_writer_test.go@@ -2,6 +2,7 @@ package poolimport ("bytes"+ "context""errors""io""sync"@@ -526,7 +527,7 @@ func TestRWConcurrency(t *testing.T) {}// Read the data back from inP and check it is OK- check := func(in io.Reader, size int64) {+ check := func(in io.Reader, size int64, rw *RW) {ck := readers.NewPatternReader(size)ckBuf := make([]byte, bufSize)rwBuf := make([]byte, bufSize)@@ -549,6 +550,7 @@ func TestRWConcurrency(t *testing.T) {if nin >= len(rwBuf) || nn >= size || inErr != io.EOF {break}+ rw.WaitWrite(context.Background())}require.Equal(t, ckBuf[:nck], rwBuf[:nin])if ckErr == io.EOF && inErr == io.EOF {@@ -560,7 +562,7 @@ func TestRWConcurrency(t *testing.T) {// Read the data back and check it is OKread := func(rw *RW, size int64) {- check(rw, size)+ check(rw, size, rw)}// Read the data back and check it is OK in using WriteTo@@ -570,7 +572,7 @@ func TestRWConcurrency(t *testing.T) {wg.Add(1)go func() {defer wg.Done()- check(in, size)+ check(in, size, rw)}()var n int64for n < size {