CasaOS - A simple, easy-to-use, elegant open-source Personal Cloud system.
added google drive and dropbox driver
link28d3ca0ca6b3
diff --git a/build/sysroot/usr/lib/systemd/system/rclone.service b/build/sysroot/usr/lib/systemd/system/rclone.servicenew file mode 100644index 0000000..92cfcab--- /dev/null+++ b/build/sysroot/usr/lib/systemd/system/rclone.service@@ -0,0 +1,11 @@+[Unit]+Description=rclone++[Service]+ExecStartPre=/usr/bin/rm -f /tmp/rclone.sock+ExecStart=/usr/bin/rclone rcd --rc-addr unix:///tmp/rclone.sock --rc-no-auth+Restart=always+RestartSec=10++[Install]+WantedBy=multi-user.target\ No newline at end of filediff --git a/build/sysroot/usr/share/casaos/shell/helper.sh b/build/sysroot/usr/share/casaos/shell/helper.shindex cfb001d..d44eceb 100644--- a/build/sysroot/usr/share/casaos/shell/helper.sh+++ b/build/sysroot/usr/share/casaos/shell/helper.sh@@ -133,11 +133,7 @@ GetPlugInDisk() {fdisk -l | grep 'Disk' | grep 'sd' | awk -F , '{print substr($1,11,3)}'}-#获取磁盘状态-#param 磁盘路径-GetDiskHealthState() {- smartctl -H $1 | grep "SMART Health Status" | awk -F ":" '{print$2}'-}+#获取磁盘字节数量和扇区数量#param 磁盘路径 /dev/sdadiff --git a/drivers/all.go b/drivers/all.goindex d27e61a..3cdfe84 100644--- a/drivers/all.go+++ b/drivers/all.go@@ -1,6 +1,7 @@package driversimport (+ _ "github.com/IceWhaleTech/CasaOS/drivers/dropbox"_ "github.com/IceWhaleTech/CasaOS/drivers/google_drive")diff --git a/drivers/dropbox/drive.go b/drivers/dropbox/drive.gonew file mode 100644index 0000000..c2aff61--- /dev/null+++ b/drivers/dropbox/drive.go@@ -0,0 +1,100 @@+package dropbox++import (+ "context"+ "errors"+ "net/http"++ "github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/internal/driver"+ "github.com/IceWhaleTech/CasaOS/model"+ "github.com/IceWhaleTech/CasaOS/pkg/utils"+ "github.com/go-resty/resty/v2"+ "go.uber.org/zap"+)++type Dropbox struct {+ model.Storage+ Addition+ AccessToken string+}++func (d *Dropbox) Config() driver.Config {+ return config+}++func (d *Dropbox) GetAddition() driver.Additional {+ return &d.Addition+}++func (d *Dropbox) Init(ctx context.Context) error {+ if len(d.RefreshToken) == 0 {+ d.getRefreshToken()+ }+ return d.refreshToken()+}++func (d *Dropbox) Drop(ctx context.Context) error {++ return nil+}++func (d *Dropbox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {+ files, err := d.getFiles(dir.GetID())+ if err != nil {+ return nil, err+ }+ return utils.SliceConvert(files, func(src File) (model.Obj, error) {+ return fileToObj(src), nil+ })+}++func (d *Dropbox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {+ url := "https://content.dropboxapi.com/2/files/download"+ link := model.Link{+ URL: url,+ Method: http.MethodPost,+ Header: http.Header{+ "Authorization": []string{"Bearer " + d.AccessToken},+ "Dropbox-API-Arg": []string{`{"path": "` + file.GetPath() + `"}`},+ },+ }+ return &link, nil+}+func (d *Dropbox) GetUserInfo(ctx context.Context) (string, error) {+ url := "https://api.dropboxapi.com/2/users/get_current_account"+ user := UserInfo{}+ resp, err := d.request(url, http.MethodPost, func(req *resty.Request) {+ req.SetHeader("Content-Type", "")+ }, &user)+ if err != nil {+ return "", err+ }+ logger.Info("resp", zap.Any("resp", string(resp)))+ return user.Email, nil+}+func (d *Dropbox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {+ return nil+}++func (d *Dropbox) Move(ctx context.Context, srcObj, dstDir model.Obj) error {+ return nil+}++func (d *Dropbox) Rename(ctx context.Context, srcObj model.Obj, newName string) error {+ return nil+}++func (d *Dropbox) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {+ return errors.New("not support")+}++func (d *Dropbox) Remove(ctx context.Context, obj model.Obj) error {+ return nil+}++func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error {+ return nil+}++var _ driver.Driver = (*Dropbox)(nil)diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.gonew file mode 100644index 0000000..ac494b3--- /dev/null+++ b/drivers/dropbox/meta.go@@ -0,0 +1,31 @@+package dropbox++import (+ "github.com/IceWhaleTech/CasaOS/internal/driver"+ "github.com/IceWhaleTech/CasaOS/internal/op"+)++const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"++type Addition struct {+ driver.RootID+ RefreshToken string `json:"refresh_token" required:"true" omit:"true"`+ AppKey string `json:"app_key" type:"string" default:"onr2ic0c0m97mxr" omit:"true"`+ AppSecret string `json:"app_secret" type:"string" default:"nd3cjtikbxyj3pz" omit:"true"`+ OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`+ AuthUrl string `json:"auth_url" type:"string" default:"https://www.dropbox.com/oauth2/authorize?client_id=onr2ic0c0m97mxr&redirect_uri=https://test-get.casaos.io&response_type=code&token_access_type=offline&state=${HOST}%2Fv1%2Frecover%2FDropbox"`+ Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"`+ Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`+}++var config = driver.Config{+ Name: "Dropbox",+ OnlyProxy: true,+ DefaultRoot: "root",+}++func init() {+ op.RegisterDriver(func() driver.Driver {+ return &Dropbox{}+ })+}diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.gonew file mode 100644index 0000000..af5bdb9--- /dev/null+++ b/drivers/dropbox/types.go@@ -0,0 +1,88 @@+package dropbox++import (+ "time"++ "github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/model"+ "go.uber.org/zap"+)++type UserInfo struct {+ AccountID string `json:"account_id"`+ Name struct {+ GivenName string `json:"given_name"`+ Surname string `json:"surname"`+ FamiliarName string `json:"familiar_name"`+ DisplayName string `json:"display_name"`+ AbbreviatedName string `json:"abbreviated_name"`+ } `json:"name"`+ Email string `json:"email"`+ EmailVerified bool `json:"email_verified"`+ Disabled bool `json:"disabled"`+ Country string `json:"country"`+ Locale string `json:"locale"`+ ReferralLink string `json:"referral_link"`+ IsPaired bool `json:"is_paired"`+ AccountType struct {+ Tag string `json:".tag"`+ } `json:"account_type"`+ RootInfo struct {+ Tag string `json:".tag"`+ RootNamespaceID string `json:"root_namespace_id"`+ HomeNamespaceID string `json:"home_namespace_id"`+ } `json:"root_info"`+}+type TokenError struct {+ Error string `json:"error"`+ ErrorDescription string `json:"error_description"`+}+type File struct {+ Tag string `json:".tag"`+ Name string `json:"name"`+ PathLower string `json:"path_lower"`+ PathDisplay string `json:"path_display"`+ ID string `json:"id"`+ ClientModified time.Time `json:"client_modified,omitempty"`+ ServerModified time.Time `json:"server_modified,omitempty"`+ Rev string `json:"rev,omitempty"`+ Size int `json:"size,omitempty"`+ IsDownloadable bool `json:"is_downloadable,omitempty"`+ ContentHash string `json:"content_hash,omitempty"`+}++type Files struct {+ Files []File `json:"entries"`+ Cursor string `json:"cursor"`+ HasMore bool `json:"has_more"`+}++type Error struct {+ Error struct {+ Errors []struct {+ Domain string `json:"domain"`+ Reason string `json:"reason"`+ Message string `json:"message"`+ LocationType string `json:"location_type"`+ Location string `json:"location"`+ }+ Code int `json:"code"`+ Message string `json:"message"`+ } `json:"error"`+}++func fileToObj(f File) *model.ObjThumb {+ logger.Info("dropbox file", zap.Any("file", f))+ obj := &model.ObjThumb{+ Object: model.Object{+ ID: f.ID,+ Name: f.Name,+ Size: int64(f.Size),+ Modified: f.ClientModified,+ IsFolder: f.Tag == "folder",+ Path: f.PathDisplay,+ },+ Thumbnail: model.Thumbnail{},+ }+ return obj+}diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.gonew file mode 100644index 0000000..da33e17--- /dev/null+++ b/drivers/dropbox/util.go@@ -0,0 +1,102 @@+package dropbox++import (+ "fmt"+ "net/http"++ "github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/drivers/base"+ "github.com/go-resty/resty/v2"+ "go.uber.org/zap"+)++func (d *Dropbox) getRefreshToken() error {+ url := "https://api.dropbox.com/oauth2/token"+ var resp base.TokenResp+ var e TokenError++ res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).+ SetFormData(map[string]string{+ "code": d.Code,+ "grant_type": "authorization_code",+ "redirect_uri": "https://test-get.casaos.io",+ }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)+ if err != nil {+ return err+ }+ logger.Info("get refresh token", zap.String("res", res.String()))+ if e.Error != "" {+ return fmt.Errorf(e.Error)+ }+ d.RefreshToken = resp.RefreshToken+ return nil++}+func (d *Dropbox) refreshToken() error {+ url := "https://api.dropbox.com/oauth2/token"+ var resp base.TokenResp+ var e TokenError++ res, err := base.RestyClient.R().SetResult(&resp).SetError(&e).+ SetFormData(map[string]string{+ "refresh_token": d.RefreshToken,+ "grant_type": "refresh_token",+ }).SetBasicAuth(d.Addition.AppKey, d.Addition.AppSecret).SetHeader("Content-Type", "application/x-www-form-urlencoded").Post(url)+ if err != nil {+ return err+ }+ logger.Info("get refresh token", zap.String("res", res.String()))+ if e.Error != "" {+ return fmt.Errorf(e.Error)+ }+ d.AccessToken = resp.AccessToken+ return nil++}+func (d *Dropbox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) {+ req := base.RestyClient.R()+ req.SetHeader("Authorization", "Bearer "+d.AccessToken)+ req.SetHeader("Content-Type", "application/json")+ if callback != nil {+ callback(req)+ }+ if resp != nil {+ req.SetResult(resp)+ }+ var e Error+ req.SetError(&e)+ res, err := req.Execute(method, url)+ if err != nil {+ return nil, err+ }+ if e.Error.Code != 0 {+ if e.Error.Code == 401 {+ err = d.refreshToken()+ if err != nil {+ return nil, err+ }+ return d.request(url, method, callback, resp)+ }+ return nil, fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors)+ }+ return res.Body(), nil+}+func (d *Dropbox) getFiles(path string) ([]File, error) {++ res := make([]File, 0)+ var resp Files+ body := base.Json{+ "limit": 2000,+ "path": path,+ }++ _, err := d.request("https://api.dropboxapi.com/2/files/list_folder", http.MethodPost, func(req *resty.Request) {+ req.SetBody(body)+ }, &resp)+ if err != nil {+ return nil, err+ }+ res = append(res, resp.Files...)++ return res, nil+}diff --git a/drivers/google_drive/drive.go b/drivers/google_drive/drive.goindex 0ad5cf0..9bad0c2 100644--- a/drivers/google_drive/drive.go+++ b/drivers/google_drive/drive.go@@ -7,11 +7,13 @@ import ("net/http""strconv"+ "github.com/IceWhaleTech/CasaOS-Common/utils/logger""github.com/IceWhaleTech/CasaOS/drivers/base""github.com/IceWhaleTech/CasaOS/internal/driver""github.com/IceWhaleTech/CasaOS/model""github.com/IceWhaleTech/CasaOS/pkg/utils""github.com/go-resty/resty/v2"+ "go.uber.org/zap")type GoogleDrive struct {@@ -33,7 +35,7 @@ func (d *GoogleDrive) Init(ctx context.Context) error {d.ChunkSize = 5}if len(d.RefreshToken) == 0 {- return d.getRefreshToken()+ d.getRefreshToken()}return d.refreshToken()}@@ -59,13 +61,24 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkAreturn nil, err}link := model.Link{- URL: url + "&alt=media",+ Method: http.MethodGet,+ URL: url + "&alt=media",Header: http.Header{"Authorization": []string{"Bearer " + d.AccessToken},},}return &link, nil}+func (d *GoogleDrive) GetUserInfo(ctx context.Context) (string, error) {+ url := "https://content.googleapis.com/drive/v3/about?fields=user"+ user := UserInfo{}+ resp, err := d.request(url, http.MethodGet, nil, &user)+ if err != nil {+ return "", err+ }+ logger.Info("resp", zap.Any("resp", resp))+ return user.User.EmailAddress, nil+}func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {data := base.Json{diff --git a/drivers/google_drive/meta.go b/drivers/google_drive/meta.goindex a617af9..16e63fb 100644--- a/drivers/google_drive/meta.go+++ b/drivers/google_drive/meta.go@@ -5,17 +5,19 @@ import ("github.com/IceWhaleTech/CasaOS/internal/op")+const ICONURL = "https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"+type Addition struct {driver.RootID- RefreshToken string `json:"refresh_token" required:"true"`- OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime"`- OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"`- ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com"`- ClientSecret string `json:"client_secret" required:"true" default:"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"`- ChunkSize int64 `json:"chunk_size" type:"number" default:"5" help:"chunk size while uploading (unit: MB)"`+ RefreshToken string `json:"refresh_token" required:"true" omit:"true"`+ OrderBy string `json:"order_by" type:"string" help:"such as: folder,name,modifiedTime" omit:"true"`+ OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" omit:"true"`+ ClientID string `json:"client_id" required:"true" default:"865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com" omit:"true"`+ ClientSecret string `json:"client_secret" required:"true" default:"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" omit:"true"`+ ChunkSize int64 `json:"chunk_size" type:"number" help:"chunk size while uploading (unit: MB)" omit:"true"`AuthUrl string `json:"auth_url" type:"string" default:"https://accounts.google.com/o/oauth2/auth/oauthchooseaccount?response_type=code&client_id=865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Ftest-get.casaos.io&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&access_type=offline&approval_prompt=force&state=${HOST}%2Fv1%2Frecover%2FGoogleDrive&service=lso&o2v=1&flowName=GeneralOAuthFlow"`Icon string `json:"icon" type:"string" default:"https://i.pcmag.com/imagery/reviews/02PHW91bUvLOs36qNbBzOiR-12.fit_scale.size_760x427.v1569471162.png"`- Code string `json:"code" type:"string" help:"code from auth_url"`+ Code string `json:"code" type:"string" help:"code from auth_url" omit:"true"`}var config = driver.Config{diff --git a/drivers/google_drive/types.go b/drivers/google_drive/types.goindex 27aaf24..4bbab52 100644--- a/drivers/google_drive/types.go+++ b/drivers/google_drive/types.go@@ -8,6 +8,17 @@ import (log "github.com/sirupsen/logrus")+type UserInfo struct {+ User struct {+ Kind string `json:"kind"`+ DisplayName string `json:"displayName"`+ PhotoLink string `json:"photoLink"`+ Me bool `json:"me"`+ PermissionID string `json:"permissionId"`+ EmailAddress string `json:"emailAddress"`+ } `json:"user"`+}+type TokenError struct {Error string `json:"error"`ErrorDescription string `json:"error_description"`diff --git a/go.mod b/go.modindex e851839..0b36533 100644--- a/go.mod+++ b/go.mod@@ -5,6 +5,7 @@ go 1.18require (github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63dgithub.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3+ github.com/Unknwon/goconfig v1.0.0github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9agithub.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcfgithub.com/deckarep/golang-set/v2 v2.1.0@@ -81,6 +82,7 @@ require (github.com/pmezard/go-difflib v1.0.0 // indirectgithub.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirectgithub.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect+ github.com/smartystreets/goconvey v1.7.2 // indirectgithub.com/tidwall/match v1.1.1 // indirectgithub.com/tidwall/pretty v1.2.0 // indirectgithub.com/tklauser/go-sysconf v0.3.11 // indirectdiff --git a/go.sum b/go.sumindex e0440dd..acc3718 100644--- a/go.sum+++ b/go.sum@@ -3,6 +3,8 @@ github.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63dgithub.com/Curtis-Milo/nat-type-identifier-go v0.0.0-20220215191915-18d42168c63d/go.mod h1:lW9x+yEjqKdPbE3+cf2fGPJXCw/hChX3Omi9QHTLFsQ=github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3 h1:jQfIty6u06fPJCutpS+97qr8uho3RpQX+B/CwHPCv/Q=github.com/IceWhaleTech/CasaOS-Common v0.4.1-alpha3/go.mod h1:xcemiRsXcs1zrmQxYMyExDjZ7UHYwkJqYE71IDIV0xA=+github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A=+github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE=github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04=github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=@@ -119,6 +121,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=github.com/googollee/go-socket.io v1.6.2 h1:olKLLHJtHz1IkL/OrTyNriZZvVQYEORNkJAqsOwPask=github.com/googollee/go-socket.io v1.6.2/go.mod h1:0vGP8/dXR9SZUMMD4+xxaGo/lohOw3YWMh2WRiWeKxg=+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=@@ -134,6 +138,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=@@ -198,6 +204,10 @@ github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY=github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=+github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=@@ -313,6 +323,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=diff --git a/internal/driver/driver.go b/internal/driver/driver.goindex 1b99c0c..b33bf80 100644--- a/internal/driver/driver.go+++ b/internal/driver/driver.go@@ -9,6 +9,7 @@ import (type Driver interface {MetaReader+ User//Writer//Other}@@ -37,7 +38,10 @@ type Reader interface {// Link get url/filepath/reader of fileLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error)}-+type User interface {+ // GetRoot get root directory of user+ GetUserInfo(ctx context.Context) (string, error)+}type Getter interface {GetRoot(ctx context.Context) (model.Obj, error)}diff --git a/internal/driver/item.go b/internal/driver/item.goindex 4d3142a..6dff49d 100644--- a/internal/driver/item.go+++ b/internal/driver/item.go@@ -40,7 +40,7 @@ type RootPath struct {}type RootID struct {- RootFolderID string `json:"root_folder_id"`+ RootFolderID string `json:"root_folder_id" omit:"true"`}func (r RootPath) GetRootPath() string {diff --git a/internal/op/driver.go b/internal/op/driver.goindex 305cb9e..ba75641 100644--- a/internal/op/driver.go+++ b/internal/op/driver.go@@ -13,7 +13,7 @@ import (type New func() driver.Drivervar driverNewMap = map[string]New{}-var driverInfoMap = map[string]driver.Info{}+var driverInfoMap = map[string][]driver.Item{} //driver.Info{}func RegisterDriver(driver New) {// log.Infof("register driver: [%s]", config.Name)@@ -39,23 +39,26 @@ func GetDriverNames() []string {return driverNames}-func GetDriverInfoMap() map[string]driver.Info {+// func GetDriverInfoMap() map[string]driver.Info {+// return driverInfoMap+// }+func GetDriverInfoMap() map[string][]driver.Item {return driverInfoMap}-func registerDriverItems(config driver.Config, addition driver.Additional) {// log.Debugf("addition of %s: %+v", config.Name, addition)tAddition := reflect.TypeOf(addition)for tAddition.Kind() == reflect.Pointer {tAddition = tAddition.Elem()}- mainItems := getMainItems(config)+ //mainItems := getMainItems(config)additionalItems := getAdditionalItems(tAddition, config.DefaultRoot)- driverInfoMap[config.Name] = driver.Info{- Common: mainItems,- Additional: additionalItems,- Config: config,- }+ driverInfoMap[config.Name] = additionalItems+ // driver.Info{+ // Common: mainItems,+ // Additional: additionalItems,+ // Config: config,+ // }}func getMainItems(config driver.Config) []driver.Item {@@ -128,6 +131,7 @@ func getMainItems(config driver.Config) []driver.Item {func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {var items []driver.Itemfor i := 0; i < t.NumField(); i++ {+field := t.Field(i)if field.Type.Kind() == reflect.Struct {items = append(items, getAdditionalItems(field.Type, defaultRoot)...)@@ -139,6 +143,9 @@ func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {if (ok1 && ignore == "true") || !ok2 {continue}+ if tag.Get("omit") == "true" {+ continue+ }item := driver.Item{Name: name,Type: strings.ToLower(field.Type.Name()),diff --git a/internal/op/fs.go b/internal/op/fs.gonew file mode 100644index 0000000..576a4b6--- /dev/null+++ b/internal/op/fs.go@@ -0,0 +1,545 @@+package op++import (+ "context"+ "os"+ stdpath "path"+ "time"++ "github.com/IceWhaleTech/CasaOS/internal/driver"+ "github.com/IceWhaleTech/CasaOS/model"+ "github.com/IceWhaleTech/CasaOS/pkg/generic_sync"+ "github.com/IceWhaleTech/CasaOS/pkg/singleflight"+ "github.com/IceWhaleTech/CasaOS/pkg/utils"+ "github.com/Xhofe/go-cache"+ "github.com/pkg/errors"+ pkgerr "github.com/pkg/errors"+ log "github.com/sirupsen/logrus"+)++// In order to facilitate adding some other things before and after file op++var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64))+var listG singleflight.Group[[]model.Obj]++func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) {+ key := Key(storage, path)+ objs, ok := listCache.Get(key)+ if ok {+ for i, obj := range objs {+ if obj.GetName() == oldObj.GetName() {+ objs[i] = newObj+ break+ }+ }+ listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))+ }+}++func delCacheObj(storage driver.Driver, path string, obj model.Obj) {+ key := Key(storage, path)+ objs, ok := listCache.Get(key)+ if ok {+ for i, oldObj := range objs {+ if oldObj.GetName() == obj.GetName() {+ objs = append(objs[:i], objs[i+1:]...)+ break+ }+ }+ listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))+ }+}++var addSortDebounceMap generic_sync.MapOf[string, func(func())]++func addCacheObj(storage driver.Driver, path string, newObj model.Obj) {+ key := Key(storage, path)+ objs, ok := listCache.Get(key)+ if ok {+ for i, obj := range objs {+ if obj.GetName() == newObj.GetName() {+ objs[i] = newObj+ return+ }+ }++ // Simple separation of files and folders+ if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() {+ objs = append(objs, newObj)+ } else {+ objs = append([]model.Obj{newObj}, objs...)+ }++ if storage.Config().LocalSort {+ debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute))+ log.Debug("addCacheObj: wait start sort")+ debounce(func() {+ log.Debug("addCacheObj: start sort")+ model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)+ addSortDebounceMap.Delete(key)+ })+ }++ listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))+ }+}++func ClearCache(storage driver.Driver, path string) {+ listCache.Del(Key(storage, path))+}++func Key(storage driver.Driver, path string) string {+ return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path))+}++// List files in storage, not contains virtual file+func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ path = utils.FixAndCleanPath(path)+ log.Debugf("op.List %s", path)+ key := Key(storage, path)+ if !utils.IsBool(refresh...) {+ if files, ok := listCache.Get(key); ok {+ log.Debugf("use cache when list %s", path)+ return files, nil+ }+ }+ dir, err := GetUnwrap(ctx, storage, path)+ if err != nil {+ return nil, errors.WithMessage(err, "failed get dir")+ }+ log.Debugf("list dir: %+v", dir)+ if !dir.IsDir() {+ return nil, errors.WithStack(errors.New("not a folder"))+ }+ objs, err, _ := listG.Do(key, func() ([]model.Obj, error) {+ files, err := storage.List(ctx, dir, args)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to list objs")+ }+ // set path+ for _, f := range files {+ if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" {+ s.SetPath(stdpath.Join(dir.GetPath(), f.GetName()))+ }+ }+ // warp obj name+ model.WrapObjsName(files)+ // call hooks+ go func(reqPath string, files []model.Obj) {+ for _, hook := range ObjsUpdateHooks {+ hook(args.ReqPath, files)+ }+ }(args.ReqPath, files)++ // sort objs+ if storage.Config().LocalSort {+ model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection)+ }+ model.ExtractFolder(files, storage.GetStorage().ExtractFolder)++ if !storage.Config().NoCache {+ if len(files) > 0 {+ log.Debugf("set cache: %s => %+v", key, files)+ listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration)))+ } else {+ log.Debugf("del cache: %s", key)+ listCache.Del(key)+ }+ }+ return files, nil+ })+ return objs, err+}++// Get object from list of files+func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {+ path = utils.FixAndCleanPath(path)+ log.Debugf("op.Get %s", path)++ // is root folder+ if utils.PathEqual(path, "/") {+ var rootObj model.Obj+ switch r := storage.GetAddition().(type) {+ case driver.IRootId:+ rootObj = &model.Object{+ ID: r.GetRootId(),+ Name: RootName,+ Size: 0,+ Modified: storage.GetStorage().Modified,+ IsFolder: true,+ Path: path,+ }+ case driver.IRootPath:+ rootObj = &model.Object{+ Path: r.GetRootPath(),+ Name: RootName,+ Size: 0,+ Modified: storage.GetStorage().Modified,+ IsFolder: true,+ }+ default:+ if storage, ok := storage.(driver.Getter); ok {+ obj, err := storage.GetRoot(ctx)+ if err != nil {+ return nil, errors.WithMessage(err, "failed get root obj")+ }+ rootObj = obj+ }+ }+ if rootObj == nil {+ return nil, errors.Errorf("please implement IRootPath or IRootId or Getter method")+ }+ return &model.ObjWrapName{+ Name: RootName,+ Obj: rootObj,+ }, nil+ }++ // not root folder+ dir, name := stdpath.Split(path)+ files, err := List(ctx, storage, dir, model.ListArgs{})+ if err != nil {+ return nil, errors.WithMessage(err, "failed get parent list")+ }+ for _, f := range files {+ // TODO maybe copy obj here+ if f.GetName() == name {+ return f, nil+ }+ }+ log.Debugf("cant find obj with name: %s", name)+ return nil, errors.WithStack(errors.New("object not found"))+}++func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) {+ obj, err := Get(ctx, storage, path)+ if err != nil {+ return nil, err+ }+ return model.UnwrapObjs(obj), err+}++var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16))+var linkG singleflight.Group[*model.Link]++// Link get link, if is an url. should have an expiry time+func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ file, err := GetUnwrap(ctx, storage, path)+ if err != nil {+ return nil, nil, errors.WithMessage(err, "failed to get file")+ }+ if file.IsDir() {+ return nil, nil, errors.WithStack(errors.New("not a file"))+ }+ key := Key(storage, path) + ":" + args.IP+ if link, ok := linkCache.Get(key); ok {+ return link, file, nil+ }+ fn := func() (*model.Link, error) {+ link, err := storage.Link(ctx, file, args)+ if err != nil {+ return nil, errors.Wrapf(err, "failed get link")+ }+ if link.Expiration != nil {+ linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration))+ }+ return link, nil+ }+ link, err, _ := linkG.Do(key, fn)+ return link, file, err+}++// Other api+func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) {+ obj, err := GetUnwrap(ctx, storage, args.Path)+ if err != nil {+ return nil, errors.WithMessagef(err, "failed to get obj")+ }+ if o, ok := storage.(driver.Other); ok {+ return o.Other(ctx, model.OtherArgs{+ Obj: obj,+ Method: args.Method,+ Data: args.Data,+ })+ } else {+ return nil, errors.New("not implement")+ }+}++var mkdirG singleflight.Group[interface{}]++func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ path = utils.FixAndCleanPath(path)+ key := Key(storage, path)+ _, err, _ := mkdirG.Do(key, func() (interface{}, error) {+ // check if dir exists+ f, err := GetUnwrap(ctx, storage, path)+ if err != nil {+ if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {+ parentPath, dirName := stdpath.Split(path)+ err = MakeDir(ctx, storage, parentPath)+ if err != nil {+ return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath)+ }+ parentDir, err := GetUnwrap(ctx, storage, parentPath)+ // this should not happen+ if err != nil {+ return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath)+ }++ switch s := storage.(type) {+ case driver.MkdirResult:+ var newObj model.Obj+ newObj, err = s.MakeDir(ctx, parentDir, dirName)+ if err == nil {+ if newObj != nil {+ addCacheObj(storage, parentPath, model.WrapObjName(newObj))+ } else if !utils.IsBool(lazyCache...) {+ ClearCache(storage, parentPath)+ }+ }+ case driver.Mkdir:+ err = s.MakeDir(ctx, parentDir, dirName)+ if err == nil && !utils.IsBool(lazyCache...) {+ ClearCache(storage, parentPath)+ }+ default:+ return nil, errors.New("not implement")+ }+ return nil, errors.WithStack(err)+ }+ return nil, errors.WithMessage(err, "failed to check if dir exists")+ }+ // dir exists+ if f.IsDir() {+ return nil, nil+ }+ // dir to make is a file+ return nil, errors.New("file exists")+ })+ return err+}++func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ srcPath = utils.FixAndCleanPath(srcPath)+ dstDirPath = utils.FixAndCleanPath(dstDirPath)+ srcRawObj, err := Get(ctx, storage, srcPath)+ if err != nil {+ return errors.WithMessage(err, "failed to get src object")+ }+ srcObj := model.UnwrapObjs(srcRawObj)+ dstDir, err := GetUnwrap(ctx, storage, dstDirPath)+ if err != nil {+ return errors.WithMessage(err, "failed to get dst dir")+ }+ srcDirPath := stdpath.Dir(srcPath)++ switch s := storage.(type) {+ case driver.MoveResult:+ var newObj model.Obj+ newObj, err = s.Move(ctx, srcObj, dstDir)+ if err == nil {+ delCacheObj(storage, srcDirPath, srcRawObj)+ if newObj != nil {+ addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))+ } else if !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ }+ case driver.Move:+ err = s.Move(ctx, srcObj, dstDir)+ if err == nil {+ delCacheObj(storage, srcDirPath, srcRawObj)+ if !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ }+ default:+ return errors.New("not implement")+ }+ return errors.WithStack(err)+}++func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ srcPath = utils.FixAndCleanPath(srcPath)+ srcRawObj, err := Get(ctx, storage, srcPath)+ if err != nil {+ return errors.WithMessage(err, "failed to get src object")+ }+ srcObj := model.UnwrapObjs(srcRawObj)+ srcDirPath := stdpath.Dir(srcPath)++ switch s := storage.(type) {+ case driver.RenameResult:+ var newObj model.Obj+ newObj, err = s.Rename(ctx, srcObj, dstName)+ if err == nil {+ if newObj != nil {+ updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj))+ } else if !utils.IsBool(lazyCache...) {+ ClearCache(storage, srcDirPath)+ }+ }+ case driver.Rename:+ err = s.Rename(ctx, srcObj, dstName)+ if err == nil && !utils.IsBool(lazyCache...) {+ ClearCache(storage, srcDirPath)+ }+ default:+ return errors.New("not implement")+ }+ return errors.WithStack(err)+}++// Copy Just copy file[s] in a storage+func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ srcPath = utils.FixAndCleanPath(srcPath)+ dstDirPath = utils.FixAndCleanPath(dstDirPath)+ srcObj, err := GetUnwrap(ctx, storage, srcPath)+ if err != nil {+ return errors.WithMessage(err, "failed to get src object")+ }+ dstDir, err := GetUnwrap(ctx, storage, dstDirPath)+ if err != nil {+ return errors.WithMessage(err, "failed to get dst dir")+ }++ switch s := storage.(type) {+ case driver.CopyResult:+ var newObj model.Obj+ newObj, err = s.Copy(ctx, srcObj, dstDir)+ if err == nil {+ if newObj != nil {+ addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))+ } else if !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ }+ case driver.Copy:+ err = s.Copy(ctx, srcObj, dstDir)+ if err == nil && !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ default:+ return errors.New("not implement")+ }+ return errors.WithStack(err)+}++func Remove(ctx context.Context, storage driver.Driver, path string) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ path = utils.FixAndCleanPath(path)+ rawObj, err := Get(ctx, storage, path)+ if err != nil {+ // if object not found, it's ok+ if errors.Is(pkgerr.Cause(err), errors.New("object not found")) {+ return nil+ }+ return errors.WithMessage(err, "failed to get object")+ }+ dirPath := stdpath.Dir(path)++ switch s := storage.(type) {+ case driver.Remove:+ err = s.Remove(ctx, model.UnwrapObjs(rawObj))+ if err == nil {+ delCacheObj(storage, dirPath, rawObj)+ }+ default:+ return errors.New("not implement")+ }+ return errors.WithStack(err)+}++func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file *model.FileStream, up driver.UpdateProgress, lazyCache ...bool) error {+ if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {+ return errors.Errorf("storage not init: %s", storage.GetStorage().Status)+ }+ defer func() {+ if f, ok := file.GetReadCloser().(*os.File); ok {+ err := os.RemoveAll(f.Name())+ if err != nil {+ log.Errorf("failed to remove file [%s]", f.Name())+ }+ }+ }()+ defer func() {+ if err := file.Close(); err != nil {+ log.Errorf("failed to close file streamer, %v", err)+ }+ }()+ // if file exist and size = 0, delete it+ dstDirPath = utils.FixAndCleanPath(dstDirPath)+ dstPath := stdpath.Join(dstDirPath, file.GetName())+ fi, err := GetUnwrap(ctx, storage, dstPath)+ if err == nil {+ if fi.GetSize() == 0 {+ err = Remove(ctx, storage, dstPath)+ if err != nil {+ return errors.WithMessagef(err, "failed remove file that exist and have size 0")+ }+ } else {+ file.Old = fi+ }+ }+ err = MakeDir(ctx, storage, dstDirPath)+ if err != nil {+ return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath)+ }+ parentDir, err := GetUnwrap(ctx, storage, dstDirPath)+ // this should not happen+ if err != nil {+ return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath)+ }+ // if up is nil, set a default to prevent panic+ if up == nil {+ up = func(p int) {}+ }++ switch s := storage.(type) {+ case driver.PutResult:+ var newObj model.Obj+ newObj, err = s.Put(ctx, parentDir, file, up)+ if err == nil {+ if newObj != nil {+ addCacheObj(storage, dstDirPath, model.WrapObjName(newObj))+ } else if !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ }+ case driver.Put:+ err = s.Put(ctx, parentDir, file, up)+ if err == nil && !utils.IsBool(lazyCache...) {+ ClearCache(storage, dstDirPath)+ }+ default:+ return errors.New("not implement")+ }+ log.Debugf("put file [%s] done", file.GetName())+ //if err == nil {+ // //clear cache+ // key := stdpath.Join(storage.GetStorage().MountPath, dstDirPath)+ // listCache.Del(key)+ //}+ return errors.WithStack(err)+}diff --git a/internal/sign/sign.go b/internal/sign/sign.gonew file mode 100644index 0000000..412bdfe--- /dev/null+++ b/internal/sign/sign.go@@ -0,0 +1,36 @@+package sign++import (+ "sync"+ "time"++ "github.com/IceWhaleTech/CasaOS/pkg/sign"+)++var once sync.Once+var instance sign.Sign++func Sign(data string) string {++ return NotExpired(data)++}++func WithDuration(data string, d time.Duration) string {+ once.Do(Instance)+ return instance.Sign(data, time.Now().Add(d).Unix())+}++func NotExpired(data string) string {+ once.Do(Instance)+ return instance.Sign(data, 0)+}++func Verify(data string, sign string) error {+ once.Do(Instance)+ return instance.Verify(data, sign)+}++func Instance() {+ instance = sign.NewHMACSign([]byte("token"))+}diff --git a/main.go b/main.goindex c63ee45..986e392 100644--- a/main.go+++ b/main.go@@ -13,6 +13,7 @@ import ("github.com/IceWhaleTech/CasaOS-Common/utils/logger""github.com/IceWhaleTech/CasaOS/pkg/cache""github.com/IceWhaleTech/CasaOS/pkg/config"+ "github.com/IceWhaleTech/CasaOS/pkg/config/configfile""github.com/IceWhaleTech/CasaOS/pkg/sqlite""github.com/IceWhaleTech/CasaOS/pkg/utils/command""github.com/IceWhaleTech/CasaOS/pkg/utils/file"@@ -69,6 +70,20 @@ func init() {service.MyService.Storages().InitStorages()route.InitFunction()+ data := &configfile.Storage{}+ e := data.Load()+ fmt.Println(e)+ fmt.Println(data.GetSectionList())+ // fmt.Println(data.HasSection("google"))+ // fmt.Println(data.GetKeyList("google"))+ // fmt.Println(data.GetValue("google", "token"))+ // data.SetValue("google", "type", "drive")+ // data.SetValue("google", "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com")+ // data.SetValue("google", "client_secret", "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")+ // data.SetValue("google", "scope", "drive")+ // data.SetValue("google", "token", `{"access_token":"ya29.a0AVvZVsqsy3vWjpjsl87mtxirrtkHpkyEXdvlORzZeIahObdEtDE47-Hzo1bIg8vJhfYKh-cdqgrUM305hiEJssFMcpkM-0IwPyxlpynMFWS0L5356AUvbv3DUd_RbV_MbKijyTThuDkfrXdLIiEOwxMOtYSXmDUaCgYKAbgSAQASFQGbdwaI6ae1NZbJARogHtpjitLGkg0166","token_type":"Bearer","refresh_token":"1//01CoIJ-aZDrUPCgYIARAAGAESNwF-L9IrNLyzp1Xzfa_sPPMouyrTgJrVchPX6uXqMizXjohTdycCpVgVcu402ND-Ikn2hArRGXA","expiry":"2023-01-28T19:26:50.198064816+08:00"}`)+ //e = data.Save()+ //fmt.Println(e)}// @title casaOS APIdiff --git a/model/args.go b/model/args.goindex b365547..4d83455 100644--- a/model/args.go+++ b/model/args.go@@ -23,6 +23,7 @@ type Link struct {Status int // status maybe 200 or 206, etcFilePath *string // local file, return the filepathExpiration *time.Duration // url expiration time+ Method string `json:"method"` // http method}type OtherArgs struct {diff --git a/model/smartctl_model.go b/model/smartctl_model.godeleted file mode 100644index ec7f75c..0000000--- a/model/smartctl_model.go+++ /dev/null@@ -1,69 +0,0 @@-package model--//-type SmartctlA struct {- Smartctl struct {- Version []int `json:"version"`- SvnRevision string `json:"svn_revision"`- PlatformInfo string `json:"platform_info"`- BuildInfo string `json:"build_info"`- Argv []string `json:"argv"`- ExitStatus int `json:"exit_status"`- } `json:"smartctl"`- Device struct {- Name string `json:"name"`- InfoName string `json:"info_name"`- Type string `json:"type"`- Protocol string `json:"protocol"`- } `json:"device"`- ModelName string `json:"model_name"`- SerialNumber string `json:"serial_number"`- FirmwareVersion string `json:"firmware_version"`- UserCapacity struct {- Blocks int `json:"blocks"`- Bytes int64 `json:"bytes"`- } `json:"user_capacity"`- SmartStatus struct {- Passed bool `json:"passed"`- } `json:"smart_status"`- AtaSmartData struct {- OfflineDataCollection struct {- Status struct {- Value int `json:"value"`- String string `json:"string"`- } `json:"status"`- CompletionSeconds int `json:"completion_seconds"`- } `json:"offline_data_collection"`- SelfTest struct {- Status struct {- Value int `json:"value"`- String string `json:"string"`- Passed bool `json:"passed"`- } `json:"status"`- PollingMinutes struct {- Short int `json:"short"`- Extended int `json:"extended"`- Conveyance int `json:"conveyance"`- } `json:"polling_minutes"`- } `json:"self_test"`- Capabilities struct {- Values []int `json:"values"`- ExecOfflineImmediateSupported bool `json:"exec_offline_immediate_supported"`- OfflineIsAbortedUponNewCmd bool `json:"offline_is_aborted_upon_new_cmd"`- OfflineSurfaceScanSupported bool `json:"offline_surface_scan_supported"`- SelfTestsSupported bool `json:"self_tests_supported"`- ConveyanceSelfTestSupported bool `json:"conveyance_self_test_supported"`- SelectiveSelfTestSupported bool `json:"selective_self_test_supported"`- AttributeAutosaveEnabled bool `json:"attribute_autosave_enabled"`- ErrorLoggingSupported bool `json:"error_logging_supported"`- GpLoggingSupported bool `json:"gp_logging_supported"`- } `json:"capabilities"`- } `json:"ata_smart_data"`- PowerOnTime struct {- Hours int `json:"hours"`- } `json:"power_on_time"`- PowerCycleCount int `json:"power_cycle_count"`- Temperature struct {- Current int `json:"current"`- } `json:"temperature"`-}diff --git a/pkg/fs/fs.go b/pkg/fs/fs.gonew file mode 100644index 0000000..ff2dee9--- /dev/null+++ b/pkg/fs/fs.go@@ -0,0 +1,12 @@+package fs++import "io"++// CheckClose is a utility function used to check the return from+// Close in a defer statement.+func CheckClose(c io.Closer, err *error) {+ cerr := c.Close()+ if *err == nil {+ *err = cerr+ }+}diff --git a/pkg/sign/hmac.go b/pkg/sign/hmac.gonew file mode 100644index 0000000..8d7f736--- /dev/null+++ b/pkg/sign/hmac.go@@ -0,0 +1,52 @@+package sign++import (+ "crypto/hmac"+ "crypto/sha256"+ "encoding/base64"+ "io"+ "strconv"+ "strings"+ "time"+)++type HMACSign struct {+ SecretKey []byte+}++func (s HMACSign) Sign(data string, expire int64) string {+ h := hmac.New(sha256.New, s.SecretKey)+ expireTimeStamp := strconv.FormatInt(expire, 10)+ _, err := io.WriteString(h, data+":"+expireTimeStamp)+ if err != nil {+ return ""+ }++ return base64.URLEncoding.EncodeToString(h.Sum(nil)) + ":" + expireTimeStamp+}++func (s HMACSign) Verify(data, sign string) error {+ signSlice := strings.Split(sign, ":")+ // check whether contains expire time+ if signSlice[len(signSlice)-1] == "" {+ return ErrExpireMissing+ }+ // check whether expire time is expired+ expires, err := strconv.ParseInt(signSlice[len(signSlice)-1], 10, 64)+ if err != nil {+ return ErrExpireInvalid+ }+ // if expire time is expired, return error+ if expires < time.Now().Unix() && expires != 0 {+ return ErrSignExpired+ }+ // verify sign+ if s.Sign(data, expires) != sign {+ return ErrSignInvalid+ }+ return nil+}++func NewHMACSign(secret []byte) Sign {+ return HMACSign{SecretKey: secret}+}diff --git a/pkg/sign/sign.go b/pkg/sign/sign.gonew file mode 100644index 0000000..2a28667--- /dev/null+++ b/pkg/sign/sign.go@@ -0,0 +1,15 @@+package sign++import "errors"++type Sign interface {+ Sign(data string, expire int64) string+ Verify(data, sign string) error+}++var (+ ErrSignExpired = errors.New("sign expired")+ ErrSignInvalid = errors.New("sign invalid")+ ErrExpireInvalid = errors.New("expire invalid")+ ErrExpireMissing = errors.New("expire missing")+)diff --git a/pkg/utils/command/command_helper.go b/pkg/utils/command/command_helper.goindex 703a8b4..1a6649e 100644--- a/pkg/utils/command/command_helper.go+++ b/pkg/utils/command/command_helper.go@@ -2,14 +2,12 @@ package commandimport ("bufio"- "context""fmt""io/ioutil""os""os/exec""path/filepath""strings"- "time")func OnlyExec(cmdStr string) {@@ -98,23 +96,6 @@ func ExecLSBLKByPath(path string) []byte {return output}-// exec smart-func ExecSmartCTLByPath(path string) []byte {- timeout := 3- ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)- defer cancel()- output, err := exec.CommandContext(ctx, "smartctl", "-a", path, "-j").Output()- if err != nil {- fmt.Println("smartctl", err)- return nil- }- return output-}--func ExecEnabledSMART(path string) {- exec.Command("smartctl", "-s on", path).Output()-}-func ExecuteScripts(scriptDirectory string) {if _, err := os.Stat(scriptDirectory); os.IsNotExist(err) {fmt.Printf("No post-start scripts at %s\n", scriptDirectory)diff --git a/pkg/utils/httper/drive.go b/pkg/utils/httper/drive.gonew file mode 100644index 0000000..b64fd1b--- /dev/null+++ b/pkg/utils/httper/drive.go@@ -0,0 +1,151 @@+package httper++import (+ "encoding/json"+ "fmt"+ "net"+ "net/http"+ "time"++ "github.com/go-resty/resty/v2"+)++type MountList struct {+ MountPoints []struct {+ MountPoint string `json:"MountPoint"`+ Fs string `json:"Fs"`+ Icon string `json:"Icon"`+ } `json:"mountPoints"`+}+type MountResult struct {+ Error string `json:"error"`+ Input struct {+ Fs string `json:"fs"`+ MountPoint string `json:"mountPoint"`+ } `json:"input"`+ Path string `json:"path"`+ Status int `json:"status"`+}++type RemotesResult struct {+ Remotes []string `json:"remotes"`+}++var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"+var DefaultTimeout = time.Second * 30++func NewRestyClient() *resty.Client {++ unixSocket := "/tmp/rclone.sock"++ transport := http.Transport{+ Dial: func(_, _ string) (net.Conn, error) {+ return net.Dial("unix", unixSocket)+ },+ }++ client := resty.New()++ client.SetTransport(&transport).SetBaseURL("http://localhost")+ client.SetRetryCount(3).SetRetryWaitTime(5*time.Second).SetTimeout(DefaultTimeout).SetHeader("User-Agent", UserAgent)+ return client+}++func GetMountList() (MountList, error) {+ var result MountList+ res, err := NewRestyClient().R().Post("/mount/listmounts")+ if err != nil {+ return result, err+ }+ if res.StatusCode() != 200 {+ return result, fmt.Errorf("get mount list failed")+ }+ json.Unmarshal(res.Body(), &result)+ for i := 0; i < len(result.MountPoints); i++ {+ result.MountPoints[i].Fs = result.MountPoints[i].Fs[:len(result.MountPoints[i].Fs)-1]+ }+ return result, err+}+func Mount(mountPoint string, fs string) error {+ res, err := NewRestyClient().R().SetFormData(map[string]string{+ "mountPoint": mountPoint,+ "fs": fs,+ }).Post("/mount/mount")+ if err != nil {+ return err+ }+ if res.StatusCode() != 200 {+ return fmt.Errorf("mount failed")+ }+ return nil+}+func Unmount(mountPoint string) error {+ res, err := NewRestyClient().R().SetFormData(map[string]string{+ "mountPoint": mountPoint,+ }).Post("/mount/unmount")+ if err != nil {+ return err+ }+ if res.StatusCode() != 200 {+ return fmt.Errorf("unmount failed")+ }+ return nil+}++func CreateConfig(data map[string]string, name, t string) error {+ data["config_is_local"] = "false"+ dataStr, _ := json.Marshal(data)+ res, err := NewRestyClient().R().SetFormData(map[string]string{+ "name": name,+ "parameters": string(dataStr),+ "type": t,+ }).Post("/config/create")+ if err != nil {+ return err+ }+ if res.StatusCode() != 200 {+ return fmt.Errorf("create config failed")+ }+ return nil+}++func GetConfigByName(name string) (map[string]string, error) {++ res, err := NewRestyClient().R().SetFormData(map[string]string{+ "name": name,+ }).Post("/config/get")+ if err != nil {+ return nil, err+ }+ if res.StatusCode() != 200 {+ return nil, fmt.Errorf("create config failed")+ }+ var result map[string]string+ json.Unmarshal(res.Body(), &result)+ return result, nil+}+func GetAllConfigName() (RemotesResult, error) {+ var result RemotesResult+ res, err := NewRestyClient().R().SetFormData(map[string]string{}).Post("/config/listremotes")+ if err != nil {+ return result, err+ }+ if res.StatusCode() != 200 {+ return result, fmt.Errorf("get config failed")+ }++ json.Unmarshal(res.Body(), &result)+ return result, nil+}+func DeleteConfigByName(name string) error {+ res, err := NewRestyClient().R().SetFormData(map[string]string{+ "name": name,+ }).Post("/config/delete")+ if err != nil {+ return err+ }+ if res.StatusCode() != 200 {+ return fmt.Errorf("delete config failed")+ }+ return nil+}diff --git a/pkg/utils/time.go b/pkg/utils/time.gonew file mode 100644index 0000000..41632e4--- /dev/null+++ b/pkg/utils/time.go@@ -0,0 +1,37 @@+package utils++import (+ "sync"+ "time"+)++func MustParseCNTime(str string) time.Time {+ lastOpTime, _ := time.ParseInLocation("2006-01-02 15:04:05 -07", str+" +08", time.Local)+ return lastOpTime+}++func NewDebounce(interval time.Duration) func(f func()) {+ var timer *time.Timer+ var lock sync.Mutex+ return func(f func()) {+ lock.Lock()+ defer lock.Unlock()+ if timer != nil {+ timer.Stop()+ }+ timer = time.AfterFunc(interval, f)+ }+}++func NewDebounce2(interval time.Duration, f func()) func() {+ var timer *time.Timer+ var lock sync.Mutex+ return func() {+ lock.Lock()+ defer lock.Unlock()+ if timer == nil {+ timer = time.AfterFunc(interval, f)+ }+ (*time.Timer)(timer).Reset(interval)+ }+}diff --git a/route/init.go b/route/init.goindex 4c55e1a..0baa719 100644--- a/route/init.go+++ b/route/init.go@@ -89,4 +89,9 @@ func InitNetworkMount() {connection.Directories = strings.Join(directories, ",")service.MyService.Connections().UpdateConnection(&connection)}++ err := service.MyService.Storage().CheckAndMountAll()+ if err != nil {+ logger.Error("mount storage err", zap.Any("err", err))+ }}diff --git a/route/route.go b/route/route.goindex aa6a476..9123cfe 100644--- a/route/route.go+++ b/route/route.go@@ -118,14 +118,8 @@ func InitRouter() *gin.Engine {v1StorageGroup.Use(){v1StorageGroup.GET("", v1.ListStorages)- v1StorageGroup.POST("", v1.CreateStorage)v1StorageGroup.DELETE("", v1.DeleteStorage)}- v1FsGroup := v1Group.Group("/fs")- v1FsGroup.Use()- {- v1FsGroup.POST("/list", v1.FsList)- }v1DriverGroup := v1Group.Group("/driver")v1DriverGroup.Use(){diff --git a/route/v1/file.go b/route/v1/file.goindex 42d0ec8..2646147 100644--- a/route/v1/file.go+++ b/route/v1/file.go@@ -1,6 +1,7 @@package v1import (+ "errors""fmt""io""io/ioutil"@@ -16,10 +17,16 @@ import ("sync""github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/internal/conf"+ "github.com/IceWhaleTech/CasaOS/internal/driver""github.com/IceWhaleTech/CasaOS/model"++ "github.com/IceWhaleTech/CasaOS/pkg/utils""github.com/IceWhaleTech/CasaOS/pkg/utils/common_err""github.com/IceWhaleTech/CasaOS/pkg/utils/file""github.com/IceWhaleTech/CasaOS/service"++ "github.com/IceWhaleTech/CasaOS/internal/sign""github.com/gin-gonic/gin"uuid "github.com/satori/go.uuid""go.uber.org/zap"@@ -190,6 +197,37 @@ func GetDownloadSingleFile(c *gin.Context) {})return}+ fileName := path.Base(filePath)+ // c.Header("Content-Disposition", "inline")+ c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName))++ storage, _ := service.MyService.FsService().GetStorage(filePath)+ if storage != nil {+ if shouldProxy(storage, fileName) {+ Proxy(c)+ return+ } else {+ link, _, err := service.MyService.FsService().Link(c, filePath, model.LinkArgs{+ IP: c.ClientIP(),+ Header: c.Request.Header,+ Type: c.Query("type"),+ })+ if err != nil {+ c.JSON(common_err.SERVICE_ERROR, model.Result{+ Success: common_err.SERVICE_ERROR,+ Message: common_err.GetMsg(common_err.SERVICE_ERROR),+ Data: err.Error(),+ })+ return++ }+ c.Header("Referrer-Policy", "no-referrer")+ c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate")+ c.Redirect(302, link.URL)+ return+ }+ }+fileTmp, err := os.Open(filePath)if err != nil {c.JSON(common_err.SERVICE_ERROR, model.Result{@@ -200,9 +238,6 @@ func GetDownloadSingleFile(c *gin.Context) {}defer fileTmp.Close()- fileName := path.Base(filePath)- // c.Header("Content-Disposition", "inline")- c.Header("Content-Disposition", "attachment; filename*=utf-8''"+url2.PathEscape(fileName))c.File(filePath)}@@ -701,3 +736,168 @@ func GetSize(c *gin.Context) {}c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: size})}+func Proxy(c *gin.Context) {+ rawPath := c.Query("path")+ filename := filepath.Base(rawPath)+ storage, err := service.MyService.FsService().GetStorage(rawPath)+ if err != nil {+ c.JSON(500, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})+ return+ }+ if canProxy(storage, filename) {+ downProxyUrl := storage.GetStorage().DownProxyUrl+ if downProxyUrl != "" {+ _, ok := c.GetQuery("d")+ if !ok {+ URL := fmt.Sprintf("%s%s?sign=%s",+ strings.Split(downProxyUrl, "\n")[0],+ utils.EncodePath(rawPath, true),+ sign.Sign(rawPath))+ c.Redirect(302, URL)+ return+ }+ }+ link, file, err := service.MyService.FsService().Link(c, rawPath, model.LinkArgs{+ Header: c.Request.Header,+ Type: c.Query("type"),+ })+ if err != nil {+ c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})++ return+ }+ err = CommonProxy(c.Writer, c.Request, link, file)+ if err != nil {+ c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})+ return+ }+ } else {+ c.JSON(common_err.SERVICE_ERROR, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: "proxy not allowed"})+ return+ }+}++// TODO need optimize+// when should be proxy?+// 1. config.MustProxy()+// 2. storage.WebProxy+// 3. proxy_types+func shouldProxy(storage driver.Driver, filename string) bool {+ if storage.Config().MustProxy() || storage.GetStorage().WebProxy {+ return true+ }+ if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) {+ return true+ }+ return false+}++// TODO need optimize+// when can be proxy?+// 1. text file+// 2. config.MustProxy()+// 3. storage.WebProxy+// 4. proxy_types+// solution: text_file + shouldProxy()+func canProxy(storage driver.Driver, filename string) bool {+ if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() {+ return true+ }+ if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) {+ return true+ }+ if utils.SliceContains(conf.SlicesMap[conf.TextTypes], utils.Ext(filename)) {+ return true+ }+ return false+}++var HttpClient = &http.Client{}++func CommonProxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error {+ // read data with native+ var err error+ if link.Data != nil {+ defer func() {+ _ = link.Data.Close()+ }()+ w.Header().Set("Content-Type", "application/octet-stream")+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))+ w.Header().Set("Content-Length", strconv.FormatInt(file.GetSize(), 10))+ if link.Header != nil {+ // TODO clean header with blacklist or whitelist+ link.Header.Del("set-cookie")+ for h, val := range link.Header {+ w.Header()[h] = val+ }+ }+ if link.Status == 0 {+ w.WriteHeader(http.StatusOK)+ } else {+ w.WriteHeader(link.Status)+ }+ _, err = io.Copy(w, link.Data)+ if err != nil {+ return err+ }+ return nil+ }+ // local file+ if link.FilePath != nil && *link.FilePath != "" {+ f, err := os.Open(*link.FilePath)+ if err != nil {+ return err+ }+ defer func() {+ _ = f.Close()+ }()+ fileStat, err := os.Stat(*link.FilePath)+ if err != nil {+ return err+ }+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))+ http.ServeContent(w, r, file.GetName(), fileStat.ModTime(), f)+ return nil+ } else {+ req, err := http.NewRequest(link.Method, link.URL, nil)+ if err != nil {+ return err+ }+ for h, val := range r.Header {+ if utils.SliceContains(conf.SlicesMap[conf.ProxyIgnoreHeaders], strings.ToLower(h)) {+ continue+ }+ req.Header[h] = val+ }+ for h, val := range link.Header {+ req.Header[h] = val+ }+ res, err := HttpClient.Do(req)+ if err != nil {+ return err+ }+ defer func() {+ _ = res.Body.Close()+ }()+ logger.Info("proxy status", zap.Any("status", res.StatusCode))+ // TODO clean header with blacklist or whitelist+ res.Header.Del("set-cookie")+ for h, v := range res.Header {+ w.Header()[h] = v+ }+ w.WriteHeader(res.StatusCode)+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.QueryEscape(file.GetName())))+ if res.StatusCode >= 400 {+ all, _ := ioutil.ReadAll(res.Body)+ msg := string(all)+ logger.Info("msg", zap.Any("msg", msg))++ return errors.New(msg)+ }+ _, err = io.Copy(w, res.Body)+ if err != nil {+ return err+ }+ return nil+ }+}diff --git a/route/v1/recover.go b/route/v1/recover.goindex df93493..76affcf 100644--- a/route/v1/recover.go+++ b/route/v1/recover.go@@ -2,41 +2,34 @@ package v1import ("strconv"+ "strings""time"- "github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/drivers/dropbox""github.com/IceWhaleTech/CasaOS/drivers/google_drive""github.com/IceWhaleTech/CasaOS/internal/op"- "github.com/IceWhaleTech/CasaOS/model""github.com/IceWhaleTech/CasaOS/service""github.com/gin-gonic/gin"- jsoniter "github.com/json-iterator/go"- "go.uber.org/zap")func GetRecoverStorage(c *gin.Context) {c.Header("Content-Type", "text/html; charset=utf-8")t := c.Param("type")+ currentTime := time.Now().UTC()+ currentDate := time.Now().UTC().Format("2006-01-02")+ // timeStr := time.Now().Format("20060102150405")if t == "GoogleDrive" {- mountPath := "google"-- mountPath += time.Now().Format("20060102150405")-gd := op.GetDriverInfoMap()[t]- var req model.Storage- req.Driver = t- req.MountPath = mountPath- req.CacheExpiration = 5add := google_drive.Addition{}add.Code = c.Query("code")if len(add.Code) == 0 {- c.String(200, `<p>code不可为空</p>`)+ c.String(200, `<p>code cannot be empty</p>`)return}add.RootFolderID = "root"- for _, v := range gd.Additional {+ for _, v := range gd {if v.Name == "client_id" {add.ClientID = v.Default}@@ -52,22 +45,119 @@ func GetRecoverStorage(c *gin.Context) {}}- var json = jsoniter.ConfigCompatibleWithStandardLibrary- addStr, err := json.Marshal(add)+ var google_drive google_drive.GoogleDrive+ google_drive.Addition = add+ err := google_drive.Init(c)+ if err != nil {+ c.String(200, `<p>Initialization failure:`+err.Error()+`</p>`)+ return+ }++ username, err := google_drive.GetUserInfo(c)+ if err != nil {+ c.String(200, `<p>Failed to get user information:`+err.Error()+`</p>`)+ return+ }+ if len(username) > 0 {+ a := strings.Split(username, "@")+ username = a[0]+ }+ username += "_drive"+ dataMap, _ := service.MyService.Storage().GetConfigByName(username)+ if len(dataMap) > 0 {+ c.String(200, `<p>The same configuration has been added</p>`)+ service.MyService.Storage().CheckAndMountByName(username)+ return+ }+ dmap := make(map[string]string)+ dmap["client_id"] = add.ClientID+ dmap["client_secret"] = add.ClientSecret+ dmap["scope"] = "drive"+ dmap["mount_point"] = "/mnt/" + username+ dmap["token"] = `{"access_token":"` + google_drive.AccessToken + `","token_type":"Bearer","refresh_token":"` + google_drive.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*1).Format("15:04:05") + `Z"}`+ // data.SetValue(username, "type", "drive")+ // data.SetValue(username, "client_id", "865173455964-4ce3gdl73ak5s15kn1vkn73htc8tant2.apps.googleusercontent.com")+ // data.SetValue(username, "client_secret", "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")+ // data.SetValue(username, "scope", "drive")+ // data.SetValue(username, "mount_point", "/mnt/"+username)+ // data.SetValue(username, "token", `{"access_token":"`+google_drive.AccessToken+`","token_type":"Bearer","refresh_token":"`+google_drive.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*1).Format("15:04:05")+`Z"}`)+ // e = data.Save()+ // if e != nil {+ // c.String(200, `<p>保存配置失败:`+e.Error()+`</p>`)+ // return+ // }+ service.MyService.Storage().CreateConfig(dmap, username, "drive")+ service.MyService.Storage().MountStorage("/mnt/"+username, username+":")+ notify := make(map[string]interface{})+ notify["status"] = "success"+ service.MyService.Notify().SendNotify("recover_status", notify)+ } else if t == "Dropbox" {++ //mountPath += timeStr++ db := op.GetDriverInfoMap()[t]++ add := dropbox.Addition{}+ add.Code = c.Query("code")+ if len(add.Code) == 0 {+ c.String(200, `<p>code cannot be empty</p>`)+ return+ }+ add.RootFolderID = ""+ for _, v := range db {+ if v.Name == "app_key" {+ add.AppKey = v.Default+ }+ if v.Name == "app_secret" {+ add.AppSecret = v.Default+ }+ }+ var dropbox dropbox.Dropbox+ dropbox.Addition = add+ err := dropbox.Init(c)if err != nil {- c.String(200, `<p>addition序列化失败</p>`)+ c.String(200, `<p>Initialization failure:`+err.Error()+`</p>`)return}- req.Addition = string(addStr)- logger.Info("GetRecoverStorage", zap.Any("req", req))- if _, err := service.MyService.Storages().CreateStorage(c, req); err != nil {- c.String(200, `<p>添加失败:`+err.Error()+`</p>`)+ username, err := dropbox.GetUserInfo(c)+ if err != nil {+ c.String(200, `<p>Failed to get user information:`+err.Error()+`</p>`)return}- data := make(map[string]interface{})- data["status"] = "success"- service.MyService.Notify().SendNotify("recover_status", data)+ if len(username) > 0 {+ a := strings.Split(username, "@")+ username = a[0]+ }+ username += "_dropbox"+ dataMap, _ := service.MyService.Storage().GetConfigByName(username)+ if len(dataMap) > 0 {+ c.String(200, `<p>The same configuration has been added</p>`)+ service.MyService.Storage().CheckAndMountByName(username)+ return+ }+ dmap := make(map[string]string)+ dmap["client_id"] = add.AppKey+ dmap["client_secret"] = add.AppSecret+ dmap["token"] = `{"access_token":"` + dropbox.AccessToken + `","token_type":"bearer","refresh_token":"` + dropbox.Addition.RefreshToken + `","expiry":"` + currentDate + `T` + currentTime.Add(time.Hour*3).Format("15:04:05") + `.780385354Z"}`+ dmap["mount_point"] = "/mnt/" + username+ // data.SetValue(username, "type", "dropbox")+ // data.SetValue(username, "client_id", add.AppKey)+ // data.SetValue(username, "client_secret", add.AppSecret)+ // data.SetValue(username, "mount_point", "/mnt/"+username)++ // data.SetValue(username, "token", `{"access_token":"`+dropbox.AccessToken+`","token_type":"bearer","refresh_token":"`+dropbox.Addition.RefreshToken+`","expiry":"`+currentDate+`T`+currentTime.Add(time.Hour*3).Format("15:04:05")+`.780385354Z"}`)+ // e = data.Save()+ // if e != nil {+ // c.String(200, `<p>保存配置失败:`+e.Error()+`</p>`)++ // return+ // }+ service.MyService.Storage().CreateConfig(dmap, username, "dropbox")+ service.MyService.Storage().MountStorage("/mnt/"+username, username+":")+ notify := make(map[string]interface{})+ notify["status"] = "success"+ service.MyService.Notify().SendNotify("recover_status", notify)}- c.String(200, `<p>关闭该页面即可</p><script>window.close()</script>`)+ c.String(200, `<p>Just close the page</p><script>window.close()</script>`)}diff --git a/route/v1/samba.go b/route/v1/samba.goindex bf3cdac..ce14f1b 100644--- a/route/v1/samba.go+++ b/route/v1/samba.go@@ -12,6 +12,7 @@ package v1import ("fmt"+ "io/ioutil""os""path/filepath""regexp"@@ -195,7 +196,10 @@ func DeleteSambaConnections(c *gin.Context) {for _, v := range mountPointList {service.MyService.Connections().UnmountSmaba(v.Path)}- os.RemoveAll(connection.MountPoint)+ dir, _ := ioutil.ReadDir(connection.MountPoint)+ if len(dir) == 0 {+ os.RemoveAll(connection.MountPoint)+ }service.MyService.Connections().DeleteConnection(id)c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: id})}diff --git a/route/v1/storage.go b/route/v1/storage.goindex 12e7d84..698a1f0 100644--- a/route/v1/storage.go+++ b/route/v1/storage.go@@ -2,8 +2,11 @@ package v1import ("strconv"+ "strings""github.com/IceWhaleTech/CasaOS-Common/utils/logger"+ "github.com/IceWhaleTech/CasaOS/drivers/dropbox"+ "github.com/IceWhaleTech/CasaOS/drivers/google_drive""github.com/IceWhaleTech/CasaOS/model""github.com/IceWhaleTech/CasaOS/pkg/utils/common_err""github.com/IceWhaleTech/CasaOS/service"@@ -12,42 +15,45 @@ import ()func ListStorages(c *gin.Context) {- var req model.PageReq- if err := c.ShouldBind(&req); err != nil {- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})- return- }- req.Validate()+ // var req model.PageReq+ // if err := c.ShouldBind(&req); err != nil {+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})+ // return+ // }+ // req.Validate()++ //logger.Info("ListStorages", zap.Any("req", req))+ //storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage)+ // if err != nil {+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})+ // return+ // }+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{+ // Content: storages,+ // Total: total,+ // }})+ r, err := service.MyService.Storage().GetStorages()- logger.Info("ListStorages", zap.Any("req", req))- storages, total, err := service.MyService.Storage().GetStorages(req.Page, req.PerPage)if err != nil {c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})return}- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: model.PageResp{- Content: storages,- Total: total,- }})-}-func CreateStorage(c *gin.Context) {- var req model.Storage- if err := c.ShouldBind(&req); err != nil {- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})- return+ for i := 0; i < len(r.MountPoints); i++ {+ dataMap, err := service.MyService.Storage().GetConfigByName(r.MountPoints[i].Fs)+ if err != nil {+ logger.Error("GetConfigByName", zap.Any("err", err))+ continue+ }+ if dataMap["type"] == "drive" {+ r.MountPoints[i].Icon = google_drive.ICONURL+ }+ if dataMap["type"] == "dropbox" {+ r.MountPoints[i].Icon = dropbox.ICONURL+ }}- if id, err := service.MyService.Storages().CreateStorage(c, req); err != nil {- data := make(map[string]interface{})- data["id"] = id- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: data})- return- } else {- data := make(map[string]interface{})- data["id"] = id- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: data})- }+ c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: r})}func UpdateStorage(c *gin.Context) {@@ -64,16 +70,19 @@ func UpdateStorage(c *gin.Context) {}func DeleteStorage(c *gin.Context) {- idStr := c.Query("id")- id, err := strconv.Atoi(idStr)- if err != nil {- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})+ json := make(map[string]string)+ c.ShouldBind(&json)+ mountPoint := json["mount_point"]+ if mountPoint == "" {+ c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: "mount_point is empty"})return}- if err := service.MyService.Storages().DeleteStorageById(c, uint(id)); err != nil {+ err := service.MyService.Storage().UnmountStorage(mountPoint)+ if err != nil {c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})return}+ service.MyService.Storage().DeleteConfigByName(strings.ReplaceAll(mountPoint, "/mnt/", ""))c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: "success"})}@@ -106,16 +115,17 @@ func EnableStorage(c *gin.Context) {}func GetStorage(c *gin.Context) {- idStr := c.Query("id")- id, err := strconv.Atoi(idStr)- if err != nil {- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})- return- }- storage, err := service.MyService.Storage().GetStorageById(uint(id))- if err != nil {- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})- return- }- c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage})++ // idStr := c.Query("id")+ // id, err := strconv.Atoi(idStr)+ // if err != nil {+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.CLIENT_ERROR, Message: common_err.GetMsg(common_err.CLIENT_ERROR), Data: err.Error()})+ // return+ // }+ // storage, err := service.MyService.Storage().GetStorageById(uint(id))+ // if err != nil {+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SERVICE_ERROR, Message: common_err.GetMsg(common_err.SERVICE_ERROR), Data: err.Error()})+ // return+ // }+ // c.JSON(common_err.SUCCESS, model.Result{Success: common_err.SUCCESS, Message: common_err.GetMsg(common_err.SUCCESS), Data: storage})}diff --git a/service/fs.go b/service/fs.goindex efd23e0..d4dc218 100644--- a/service/fs.go+++ b/service/fs.go@@ -6,12 +6,14 @@ import ("github.com/IceWhaleTech/CasaOS-Common/utils/logger""github.com/IceWhaleTech/CasaOS/internal/driver""github.com/IceWhaleTech/CasaOS/model"+ log "github.com/dsoprea/go-logging""go.uber.org/zap")type FsService interface {FList(ctx context.Context, path string, refresh ...bool) ([]model.Obj, error)GetStorage(path string) (driver.Driver, error)+ Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error)}type fsService struct {@@ -39,14 +41,14 @@ func (f *fsService) FList(ctx context.Context, path string, refresh ...bool) ([]// return res, nil// }-// func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {-// res, file, err := link(ctx, path, args)-// if err != nil {-// log.Errorf("failed link %s: %+v", path, err)-// return nil, nil, err-// }-// return res, file, nil-// }+func (f *fsService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {+ res, file, err := MyService.FsLinkService().Link(ctx, path, args)+ if err != nil {+ log.Errorf("failed link %s: %+v", path, err)+ return nil, nil, err+ }+ return res, file, nil+}// func (f *fsService) MakeDir(ctx context.Context, path string, lazyCache ...bool) error {// err := makeDir(ctx, path, lazyCache...)diff --git a/service/fs_link.go b/service/fs_link.gonew file mode 100644index 0000000..bf4c013--- /dev/null+++ b/service/fs_link.go@@ -0,0 +1,27 @@+package service++import (+ "context"++ "github.com/IceWhaleTech/CasaOS/internal/op"+ "github.com/IceWhaleTech/CasaOS/model"+ "github.com/pkg/errors"+)++type FsLinkService interface {+ Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error)+}++type fsLinkService struct {+}++func (f *fsLinkService) Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, model.Obj, error) {+ storage, actualPath, err := MyService.StoragePath().GetStorageAndActualPath(path)+ if err != nil {+ return nil, nil, errors.WithMessage(err, "failed get storage")+ }+ return op.Link(ctx, storage, actualPath, args)+}+func NewFsLinkService() FsLinkService {+ return &fsLinkService{}+}diff --git a/service/service.go b/service/service.goindex 2d505a4..5c08b53 100644--- a/service/service.go+++ b/service/service.go@@ -42,6 +42,7 @@ type Repository interface {Storages() StoragesServiceStoragePath() StoragePathServiceFsListService() FsListService+ FsLinkService() FsLinkServiceFsService() FsService}@@ -63,10 +64,11 @@ func NewService(db *gorm.DB, RuntimePath string, socket *socketio.Server) Reposisystem: NewSystemService(),shares: NewSharesService(db),connections: NewConnectionsService(db),- storage: NewStorageService(db),+ storage: NewStorageService(),storages: NewStoragesService(),storage_path: NewStoragePathService(),fs_list: NewFsListService(),+ fs_link: NewFsLinkService(),fs: NewFsService(),}}@@ -84,9 +86,13 @@ type store struct {storages StoragesServicestorage_path StoragePathServicefs_list FsListService+ fs_link FsLinkServicefs FsService}+func (c *store) FsLinkService() FsLinkService {+ return c.fs_link+}func (c *store) FsService() FsService {return c.fs}diff --git a/service/storage.go b/service/storage.goindex 9ef53e3..cf347f1 100644--- a/service/storage.go+++ b/service/storage.go@@ -1,73 +1,100 @@package serviceimport (- "fmt"+ "io/ioutil"- "github.com/IceWhaleTech/CasaOS/model"- "github.com/pkg/errors"- "gorm.io/gorm"+ "github.com/IceWhaleTech/CasaOS/pkg/utils/file"+ "github.com/IceWhaleTech/CasaOS/pkg/utils/httper")type StorageService interface {- CreateStorage(storage *model.Storage) error- UpdateStorage(storage *model.Storage) error- DeleteStorageById(id uint) error- GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error)- GetStorageById(id uint) (*model.Storage, error)- GetEnabledStorages() ([]model.Storage, error)+ MountStorage(mountPoint, fs string) error+ UnmountStorage(mountPoint string) error+ GetStorages() (httper.MountList, error)+ CreateConfig(data map[string]string, name string, t string) error+ CheckAndMountByName(name string) error+ CheckAndMountAll() error+ GetConfigByName(name string) (map[string]string, error)+ DeleteConfigByName(name string) error}type storageStruct struct {- db *gorm.DB}-// CreateStorage just insert storage to database-func (s *storageStruct) CreateStorage(storage *model.Storage) error {- return errors.WithStack(s.db.Create(storage).Error)+func (s *storageStruct) MountStorage(mountPoint, fs string) error {+ file.IsNotExistMkDir(mountPoint)+ httper.Mount(mountPoint, fs)+ return nil}+func (s *storageStruct) UnmountStorage(mountPoint string) error {+ err := httper.Unmount(mountPoint)+ if err == nil {+ dir, _ := ioutil.ReadDir(mountPoint)-// UpdateStorage just update storage in database-func (s *storageStruct) UpdateStorage(storage *model.Storage) error {- return errors.WithStack(s.db.Save(storage).Error)+ if len(dir) == 0 {+ file.RMDir(mountPoint)+ }+ return nil+ }+ return err}--// DeleteStorageById just delete storage from database by id-func (s *storageStruct) DeleteStorageById(id uint) error {- return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error)+func (s *storageStruct) GetStorages() (httper.MountList, error) {+ return httper.GetMountList()}--// GetStorages Get all storages from database order by index-func (s *storageStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {- storageDB := s.db.Model(&model.Storage{})- var count int64- if err := storageDB.Count(&count).Error; err != nil {- return nil, 0, errors.Wrapf(err, "failed get storages count")+func (s *storageStruct) CreateConfig(data map[string]string, name string, t string) error {+ httper.CreateConfig(data, name, t)+ return nil+}+func (s *storageStruct) CheckAndMountByName(name string) error {+ storages, _ := MyService.Storage().GetStorages()+ currentRemote, _ := httper.GetConfigByName(name)+ mountPoint := currentRemote["mount_point"]+ isMount := false+ for _, v := range storages.MountPoints {+ if v.MountPoint == mountPoint {+ isMount = true+ break+ }}- var storages []model.Storage- if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil {- return nil, 0, errors.WithStack(err)+ if !isMount {+ MyService.Storage().MountStorage(mountPoint, name+":")}- return storages, count, nil+ return nil}--// GetStorageById Get Storage by id, used to update storage usually-func (s *storageStruct) GetStorageById(id uint) (*model.Storage, error) {- var storage model.Storage- storage.ID = id- if err := s.db.First(&storage).Error; err != nil {- return nil, errors.WithStack(err)+func (s *storageStruct) CheckAndMountAll() error {+ storages, err := MyService.Storage().GetStorages()+ if err != nil {+ return err}- return &storage, nil-}--func (s *storageStruct) GetEnabledStorages() ([]model.Storage, error) {- var storages []model.Storage- if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil {- return nil, errors.WithStack(err)+ section, err := httper.GetAllConfigName()+ if err != nil {+ return err+ }+ for _, v := range section.Remotes {+ currentRemote, _ := httper.GetConfigByName(v)+ mountPoint := currentRemote["mount_point"]+ if len(mountPoint) == 0 {+ continue+ }+ isMount := false+ for _, v := range storages.MountPoints {+ if v.MountPoint == mountPoint {+ isMount = true+ break+ }+ }+ if !isMount {+ return MyService.Storage().MountStorage(mountPoint, v+":")+ }}- return storages, nil+ return nil}--func NewStorageService(db *gorm.DB) StorageService {- return &storageStruct{db: db}+func (s *storageStruct) GetConfigByName(name string) (map[string]string, error) {+ return httper.GetConfigByName(name)+}+func (s *storageStruct) DeleteConfigByName(name string) error {+ return httper.DeleteConfigByName(name)+}+func NewStorageService() StorageService {+ return &storageStruct{}}diff --git a/service/storage_old.go b/service/storage_old.gonew file mode 100644index 0000000..d5a928a--- /dev/null+++ b/service/storage_old.go@@ -0,0 +1,73 @@+package service++import (+ "fmt"++ "github.com/IceWhaleTech/CasaOS/model"+ "github.com/pkg/errors"+ "gorm.io/gorm"+)++type StorageOldService interface {+ CreateStorage(storage *model.Storage) error+ UpdateStorage(storage *model.Storage) error+ DeleteStorageById(id uint) error+ GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error)+ GetStorageById(id uint) (*model.Storage, error)+ GetEnabledStorages() ([]model.Storage, error)+}++type storageOldStruct struct {+ db *gorm.DB+}++// CreateStorage just insert storage to database+func (s *storageOldStruct) CreateStorage(storage *model.Storage) error {+ return errors.WithStack(s.db.Create(storage).Error)+}++// UpdateStorage just update storage in database+func (s *storageOldStruct) UpdateStorage(storage *model.Storage) error {+ return errors.WithStack(s.db.Save(storage).Error)+}++// DeleteStorageById just delete storage from database by id+func (s *storageOldStruct) DeleteStorageById(id uint) error {+ return errors.WithStack(s.db.Delete(&model.Storage{}, id).Error)+}++// GetStorages Get all storages from database order by index+func (s *storageOldStruct) GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) {+ storageDB := s.db.Model(&model.Storage{})+ var count int64+ if err := storageDB.Count(&count).Error; err != nil {+ return nil, 0, errors.Wrapf(err, "failed get storages count")+ }+ var storages []model.Storage+ if err := storageDB.Order("`order`").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil {+ return nil, 0, errors.WithStack(err)+ }+ return storages, count, nil+}++// GetStorageById Get Storage by id, used to update storage usually+func (s *storageOldStruct) GetStorageById(id uint) (*model.Storage, error) {+ var storage model.Storage+ storage.ID = id+ if err := s.db.First(&storage).Error; err != nil {+ return nil, errors.WithStack(err)+ }+ return &storage, nil+}++func (s *storageOldStruct) GetEnabledStorages() ([]model.Storage, error) {+ var storages []model.Storage+ if err := s.db.Where(fmt.Sprintf("%s = ?", "disabled"), false).Find(&storages).Error; err != nil {+ return nil, errors.WithStack(err)+ }+ return storages, nil+}++func NewStorageOldService(db *gorm.DB) StorageOldService {+ return &storageOldStruct{db: db}+}diff --git a/service/storage_service.go b/service/storage_service.goindex 89ed619..30cfeed 100644--- a/service/storage_service.go+++ b/service/storage_service.go@@ -15,7 +15,6 @@ import ("github.com/IceWhaleTech/CasaOS/model"- "github.com/IceWhaleTech/CasaOS/internal/conf""github.com/IceWhaleTech/CasaOS/internal/driver""github.com/IceWhaleTech/CasaOS/internal/op"mapset "github.com/deckarep/golang-set/v2"@@ -30,9 +29,9 @@ type StoragesService interface {DisableStorage(ctx context.Context, id uint) errorUpdateStorage(ctx context.Context, storage model.Storage) errorDeleteStorageById(ctx context.Context, id uint) error- MustSaveDriverStorage(driver driver.Driver)+ MustSaveDriverStorage(driver driver.Driver) errorGetStorageVirtualFilesByPath(prefix string) []model.Obj- initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error)+ initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error)InitStorages()GetBalancedStorage(path string) driver.Driver}@@ -75,18 +74,28 @@ func (s *storagesStruct) CreateStorage(ctx context.Context, storage model.Storagreturn 0, errors.WithMessage(err, "failed get driver new")}storageDriver := driverNew()- // insert storage to database- err = MyService.Storage().CreateStorage(&storage)+ // // insert storage to database+ // err = MyService.Storage().CreateStorage(&storage)+ // if err != nil {++ // return storage.ID, errors.WithMessage(err, "failed create storage in database")+ // }+ // already has an id+ err = s.initStorage(ctx, storage, storageDriver, func(d driver.Driver, ctx context.Context) string {+ u, _ := d.GetUserInfo(ctx)+ if len(u) > 0 {+ a := strings.Split(u, "@")+ u = a[0]+ }+ return u+ })if err != nil {- return storage.ID, errors.WithMessage(err, "failed create storage in database")+ s.DeleteStorageById(ctx, storage.ID)+ return storage.ID, errors.Wrap(err, "failed init storage")}- // already has an id- err = s.initStorage(ctx, storage, storageDriver)+go op.CallStorageHooks("add", storageDriver)- if err != nil {- return storage.ID, errors.Wrap(err, "failed init storage but storage is already created")- }logger.Error("storage created", zap.Any("storage", storageDriver))return storage.ID, nil}@@ -102,14 +111,14 @@ func (s *storagesStruct) LoadStorage(ctx context.Context, storage model.Storage)}storageDriver := driverNew()- err = s.initStorage(ctx, storage, storageDriver)+ err = s.initStorage(ctx, storage, storageDriver, nil)go op.CallStorageHooks("add", storageDriver)logger.Info("storage created", zap.Any("storage", storageDriver))return err}// initStorage initialize the driver and store to storagesMap-func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) {+func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver, setMountPath func(d driver.Driver, ctx context.Context) string) (err error) {storageDriver.SetStorage(storage)driverStorage := storageDriver.GetStorage()@@ -121,61 +130,72 @@ func (s *storagesStruct) initStorage(ctx context.Context, storage model.Storage,if err == nil {err = storageDriver.Init(ctx)}+ if setMountPath != nil {+ driverStorage.MountPath += "_" + setMountPath(storageDriver, ctx)++ }+ if s.HasStorage(driverStorage.MountPath) {+ return errors.New("mount path already exists")+ }+ storageDriver.SetStorage(*driverStorage)storagesMap.Store(driverStorage.MountPath, storageDriver)+if err != nil {driverStorage.SetStatus(err.Error())err = errors.Wrap(err, "failed init storage")} else {driverStorage.SetStatus(op.WORK)}- s.MustSaveDriverStorage(storageDriver)++ err = s.MustSaveDriverStorage(storageDriver)+return err}func (s *storagesStruct) EnableStorage(ctx context.Context, id uint) error {- storage, err := MyService.Storage().GetStorageById(id)- if err != nil {- return errors.WithMessage(err, "failed get storage")- }- if !storage.Disabled {- return errors.Errorf("this storage have enabled")- }- storage.Disabled = false- err = MyService.Storage().UpdateStorage(storage)- if err != nil {- return errors.WithMessage(err, "failed update storage in db")- }- err = s.LoadStorage(ctx, *storage)- if err != nil {- return errors.WithMessage(err, "failed load storage")- }+ // storage, err := MyService.Storage().GetStorageById(id)+ // if err != nil {+ // return errors.WithMessage(err, "failed get storage")+ // }+ // if !storage.Disabled {+ // return errors.Errorf("this storage have enabled")+ // }+ // storage.Disabled = false+ // err = MyService.Storage().UpdateStorage(storage)+ // if err != nil {+ // return errors.WithMessage(err, "failed update storage in db")+ // }+ // err = s.LoadStorage(ctx, *storage)+ // if err != nil {+ // return errors.WithMessage(err, "failed load storage")+ // }return nil}func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error {- storage, err := MyService.Storage().GetStorageById(id)- if err != nil {- return errors.WithMessage(err, "failed get storage")- }- if storage.Disabled {- return errors.Errorf("this storage have disabled")- }- storageDriver, err := GetStorageByMountPath(storage.MountPath)- if err != nil {- return errors.WithMessage(err, "failed get storage driver")- }- // drop the storage in the driver- if err := storageDriver.Drop(ctx); err != nil {- return errors.Wrap(err, "failed drop storage")- }- // delete the storage in the memory- storage.Disabled = true- err = MyService.Storage().UpdateStorage(storage)- if err != nil {- return errors.WithMessage(err, "failed update storage in db")- }- storagesMap.Delete(storage.MountPath)- go op.CallStorageHooks("del", storageDriver)+ // storage, err := MyService.Storage().GetStorageById(id)+ // if err != nil {+ // return errors.WithMessage(err, "failed get storage")+ // }+ // if storage.Disabled {+ // return errors.Errorf("this storage have disabled")+ // }+ // storageDriver, err := GetStorageByMountPath(storage.MountPath)+ // if err != nil {+ // return errors.WithMessage(err, "failed get storage driver")+ // }+ // // drop the storage in the driver+ // if err := storageDriver.Drop(ctx); err != nil {+ // return errors.Wrap(err, "failed drop storage")+ // }+ // // delete the storage in the memory+ // storage.Disabled = true+ // err = MyService.Storage().UpdateStorage(storage)+ // if err != nil {+ // return errors.WithMessage(err, "failed update storage in db")+ // }+ // storagesMap.Delete(storage.MountPath)+ // go op.CallStorageHooks("del", storageDriver)return nil}@@ -183,90 +203,92 @@ func (s *storagesStruct) DisableStorage(ctx context.Context, id uint) error {// get old storage first// drop the storage then reinitializefunc (s *storagesStruct) UpdateStorage(ctx context.Context, storage model.Storage) error {- oldStorage, err := MyService.Storage().GetStorageById(storage.ID)- if err != nil {- return errors.WithMessage(err, "failed get old storage")- }- if oldStorage.Driver != storage.Driver {- return errors.Errorf("driver cannot be changed")- }- storage.Modified = time.Now()- storage.MountPath = utils.FixAndCleanPath(storage.MountPath)- err = MyService.Storage().UpdateStorage(&storage)- if err != nil {- return errors.WithMessage(err, "failed update storage in database")- }- if storage.Disabled {- return nil- }- storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)- if oldStorage.MountPath != storage.MountPath {- // mount path renamed, need to drop the storage- storagesMap.Delete(oldStorage.MountPath)- }- if err != nil {- return errors.WithMessage(err, "failed get storage driver")- }- err = storageDriver.Drop(ctx)- if err != nil {- return errors.Wrapf(err, "failed drop storage")- }-- err = s.initStorage(ctx, storage, storageDriver)- go op.CallStorageHooks("update", storageDriver)-- logger.Info("storage updated", zap.Any("storage", storageDriver))- return err+ // oldStorage, err := MyService.Storage().GetStorageById(storage.ID)+ // if err != nil {+ // return errors.WithMessage(err, "failed get old storage")+ // }+ // if oldStorage.Driver != storage.Driver {+ // return errors.Errorf("driver cannot be changed")+ // }+ // storage.Modified = time.Now()+ // storage.MountPath = utils.FixAndCleanPath(storage.MountPath)+ // err = MyService.Storage().UpdateStorage(&storage)+ // if err != nil {+ // return errors.WithMessage(err, "failed update storage in database")+ // }+ // if storage.Disabled {+ // return nil+ // }+ // storageDriver, err := GetStorageByMountPath(oldStorage.MountPath)+ // if oldStorage.MountPath != storage.MountPath {+ // // mount path renamed, need to drop the storage+ // storagesMap.Delete(oldStorage.MountPath)+ // }+ // if err != nil {+ // return errors.WithMessage(err, "failed get storage driver")+ // }+ // err = storageDriver.Drop(ctx)+ // if err != nil {+ // return errors.Wrapf(err, "failed drop storage")+ // }++ // err = s.initStorage(ctx, storage, storageDriver, nil)+ // go op.CallStorageHooks("update", storageDriver)++ // logger.Info("storage updated", zap.Any("storage", storageDriver))+ //return err+ return nil}func (s *storagesStruct) DeleteStorageById(ctx context.Context, id uint) error {- storage, err := MyService.Storage().GetStorageById(id)- if err != nil {- return errors.WithMessage(err, "failed get storage")- }- if !storage.Disabled {- storageDriver, err := GetStorageByMountPath(storage.MountPath)- if err != nil {- return errors.WithMessage(err, "failed get storage driver")- }- // drop the storage in the driver- if err := storageDriver.Drop(ctx); err != nil {- return errors.Wrapf(err, "failed drop storage")- }- // delete the storage in the memory- storagesMap.Delete(storage.MountPath)- go op.CallStorageHooks("del", storageDriver)- }- // delete the storage in the database- if err := MyService.Storage().DeleteStorageById(id); err != nil {- return errors.WithMessage(err, "failed delete storage in database")- }+ // storage, err := MyService.Storage().GetStorageById(id)+ // if err != nil {+ // return errors.WithMessage(err, "failed get storage")+ // }+ // if !storage.Disabled {+ // storageDriver, err := GetStorageByMountPath(storage.MountPath)+ // if err == nil {+ // // drop the storage in the driver+ // if err := storageDriver.Drop(ctx); err != nil {+ // return errors.Wrapf(err, "failed drop storage")+ // }+ // // delete the storage in the memory+ // storagesMap.Delete(storage.MountPath)+ // }++ // go op.CallStorageHooks("del", storageDriver)+ // }+ // // delete the storage in the database+ // if err := MyService.Storage().DeleteStorageById(id); err != nil {+ // return errors.WithMessage(err, "failed delete storage in database")+ // }return nil}// MustSaveDriverStorage call from specific driver-func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) {+func (s *storagesStruct) MustSaveDriverStorage(driver driver.Driver) error {err := saveDriverStorage(driver)if err != nil {logger.Error("failed save driver storage", zap.Any("err", err))}+ return err}func saveDriverStorage(driver driver.Driver) error {- storage := driver.GetStorage()- addition := driver.GetAddition()-- var json = jsoniter.ConfigCompatibleWithStandardLibrary-- str, err := json.MarshalToString(addition)- if err != nil {- return errors.Wrap(err, "error while marshal addition")- }- storage.Addition = str- err = MyService.Storage().UpdateStorage(storage)- if err != nil {- return errors.WithMessage(err, "failed update storage in database")- }+ // storage := driver.GetStorage()+ // addition := driver.GetAddition()++ // var json = jsoniter.ConfigCompatibleWithStandardLibrary++ // str, err := json.MarshalToString(addition)+ // if err != nil {+ // return errors.Wrap(err, "error while marshal addition")+ // }+ // storage.Addition = str+ // err = MyService.Storage().UpdateStorage(storage)+ // if err != nil {+ // return errors.WithMessage(err, "failed update storage in database")+ // }return nil}@@ -354,21 +376,21 @@ func (s *storagesStruct) GetBalancedStorage(path string) driver.Driver {}}func (s *storagesStruct) InitStorages() {- storages, err := MyService.Storage().GetEnabledStorages()- if err != nil {- logger.Error("failed get enabled storages", zap.Any("err", err))- }- go func(storages []model.Storage) {- for i := range storages {- err := s.LoadStorage(context.Background(), storages[i])- if err != nil {- logger.Error("failed get enabled storages", zap.Any("err", err))- } else {- logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath))- }- }- conf.StoragesLoaded = true- }(storages)+ // storages, err := MyService.Storage().GetEnabledStorages()+ // if err != nil {+ // logger.Error("failed get enabled storages", zap.Any("err", err))+ // }+ // go func(storages []model.Storage) {+ // for i := range storages {+ // err := s.LoadStorage(context.Background(), storages[i])+ // if err != nil {+ // logger.Error("failed get enabled storages", zap.Any("err", err))+ // } else {+ // logger.Info("success load storage", zap.String("mount_path", storages[i].MountPath))+ // }+ // }+ // conf.StoragesLoaded = true+ // }(storages)}func NewStoragesService() StoragesService {