Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bin: add bisync.md generator

nielashfee471634386
diff --git a/Makefile b/Makefile
index 05f7bc28d..03242ef8d 100644
--- a/Makefile
+++ b/Makefile
@@ -149,6 +149,7 @@ commanddocs: rclone
-@rmdir -p '$$HOME/.config/rclone'
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
+ go run bin/make_bisync_docs.go ./docs/content/
backenddocs: rclone bin/make_backend_docs.py
-@rmdir -p '$$HOME/.config/rclone'
diff --git a/bin/make_bisync_docs.go b/bin/make_bisync_docs.go
new file mode 100644
index 000000000..3f0dc125c
--- /dev/null
+++ b/bin/make_bisync_docs.go
@@ -0,0 +1,159 @@
+//go:build ignore
+
+package main
+
+import (
+ "bytes"
+ "cmp"
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fstest/runs"
+ "github.com/stretchr/testify/assert/yaml"
+)
+
+var path = flag.String("path", "./docs/content/", "root path")
+
+const (
+ configFile = "fstest/test_all/config.yaml"
+ startListIgnores = "<!--- start list_ignores - DO NOT EDIT THIS SECTION - use make commanddocs --->"
+ endListIgnores = "<!--- end list_ignores - DO NOT EDIT THIS SECTION - use make commanddocs --->"
+ startListFailures = "<!--- start list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->"
+ endListFailures = "<!--- end list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->"
+ integrationTestsJSONURL = "https://pub.rclone.org/integration-tests/current/index.json"
+ integrationTestsHTMLURL = "https://pub.rclone.org/integration-tests/current/"
+)
+
+func main() {
+ err := replaceBetween(*path, startListIgnores, endListIgnores, getIgnores)
+ if err != nil {
+ fs.Errorf(*path, "error replacing ignores: %v", err)
+ }
+ err = replaceBetween(*path, startListFailures, endListFailures, getFailures)
+ if err != nil {
+ fs.Errorf(*path, "error replacing failures: %v", err)
+ }
+}
+
+// replaceBetween replaces the text between startSep and endSep with fn()
+func replaceBetween(path, startSep, endSep string, fn func() (string, error)) error {
+ b, err := os.ReadFile(filepath.Join(path, "bisync.md"))
+ if err != nil {
+ return err
+ }
+ doc := string(b)
+
+ before, after, found := strings.Cut(doc, startSep)
+ if !found {
+ return fmt.Errorf("could not find: %v", startSep)
+ }
+ _, after, found = strings.Cut(after, endSep)
+ if !found {
+ return fmt.Errorf("could not find: %v", endSep)
+ }
+
+ replaceSection, err := fn()
+ if err != nil {
+ return err
+ }
+
+ newDoc := before + startSep + "\n" + strings.TrimSpace(replaceSection) + "\n" + endSep + after
+
+ err = os.WriteFile(filepath.Join(path, "bisync.md"), []byte(newDoc), 0777)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// getIgnores updates the list of ignores from config.yaml
+func getIgnores() (string, error) {
+ config, err := parseConfig()
+ if err != nil {
+ return "", fmt.Errorf("failed to parse config: %v", err)
+ }
+ s := ""
+ slices.SortFunc(config.Backends, func(a, b runs.Backend) int {
+ return cmp.Compare(a.Remote, b.Remote)
+ })
+ for _, backend := range config.Backends {
+ include := false
+
+ if slices.Contains(backend.IgnoreTests, "cmd/bisync") {
+ include = true
+ s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(backend.Remote, ":"), backend.Backend)
+ }
+
+ for _, ignore := range backend.Ignore {
+ if strings.Contains(strings.ToLower(ignore), "bisync") {
+ if !include { // don't have header row yet
+ s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(backend.Remote, ":"), backend.Backend)
+ }
+ include = true
+ s += fmt.Sprintf(" - `%s`\n", ignore)
+ // TODO: might be neat to add a "reason" param displaying the reason the test is ignored
+ }
+ }
+ }
+ return s, nil
+}
+
+// getFailures updates the list of currently failing tests from the integration tests server
+func getFailures() (string, error) {
+ var buf bytes.Buffer
+ err := operations.CopyURLToWriter(context.Background(), integrationTestsJSONURL, &buf)
+ if err != nil {
+ return "", err
+ }
+
+ r := runs.Report{}
+ err = json.Unmarshal(buf.Bytes(), &r)
+ if err != nil {
+ return "", fmt.Errorf("failed to unmarshal json: %v", err)
+ }
+
+ s := ""
+ for _, run := range r.Failed {
+ for i, t := range run.FailedTests {
+ if strings.Contains(strings.ToLower(t), "bisync") {
+
+ if i == 0 { // don't have header row yet
+ s += fmt.Sprintf("- `%s` (`%s`)\n", strings.TrimSuffix(run.Remote, ":"), run.Backend)
+ }
+
+ url := integrationTestsHTMLURL + run.TrialName
+ url = url[:len(url)-5] + "1.txt" // numbers higher than 1 could change from night to night
+ s += fmt.Sprintf(" - [`%s`](%v)\n", t, url)
+
+ if i == 4 && len(run.FailedTests) > 5 { // stop after 5
+ s += fmt.Sprintf(" - [%v more](%v)\n", len(run.FailedTests)-5, integrationTestsHTMLURL)
+ break
+ }
+ }
+ }
+ }
+ s += fmt.Sprintf("- Updated: %v", r.DateTime)
+ return s, nil
+}
+
+// parseConfig reads and parses the config.yaml file
+func parseConfig() (*runs.Config, error) {
+ d, err := os.ReadFile(configFile)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read config file: %w", err)
+ }
+ config := &runs.Config{}
+ err = yaml.Unmarshal(d, &config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse config file: %w", err)
+ }
+ return config, nil
+}
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 2473b58ea..a72bc8e24 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -1046,7 +1046,7 @@ encodings.)
The following backends have known issues that need more investigation:
-<!--- start list_failures - DO NOT EDIT THIS SECTION - use rclone gendocs --->
+<!--- start list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
- `TestGoFile` (`gofile`)
- [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt)
- [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt)
@@ -1055,12 +1055,12 @@ The following backends have known issues that need more investigation:
- [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/gofile-cmd.bisync-TestGoFile-1.txt)
- [78 more](https://pub.rclone.org/integration-tests/current/)
- Updated: 2025-08-21-010015
-<!--- end list_failures - DO NOT EDIT THIS SECTION - use rclone gendocs --->
+<!--- end list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
The following backends either have not been tested recently or have known issues
that are deemed unfixable for the time being:
-<!--- start list_ignores - DO NOT EDIT THIS SECTION - use rclone gendocs --->
+<!--- start list_ignores - DO NOT EDIT THIS SECTION - use make commanddocs --->
- `TestCache` (`cache`)
- `TestFileLu` (`filelu`)
- `TestFilesCom` (`filescom`)
@@ -1085,7 +1085,7 @@ that are deemed unfixable for the time being:
- `TestWebdavNextcloud` (`webdav`)
- `TestWebdavOwncloud` (`webdav`)
- `TestnStorage` (`netstorage`)
-<!--- end list_ignores - DO NOT EDIT THIS SECTION - use rclone gendocs --->
+<!--- end list_ignores - DO NOT EDIT THIS SECTION - use make commanddocs --->
([more info](https://github.com/rclone/rclone/blob/master/fstest/test_all/config.yaml))
The above lists are updated for each stable release of rclone. For test results