A Q&A platform software for teams at any scales. Whether it's a community forum, help center, or knowledge management platform, you can always count on Apache Answer.
feat(service): implement file cleanup and deletion functionality
LinkinStars5cffab864939
diff --git a/cmd/wire_gen.go b/cmd/wire_gen.goindex b44fadfb..702f60df 100644--- a/cmd/wire_gen.go+++ b/cmd/wire_gen.go@@ -48,6 +48,7 @@ import ("github.com/apache/answer/internal/repo/comment""github.com/apache/answer/internal/repo/config""github.com/apache/answer/internal/repo/export"+ "github.com/apache/answer/internal/repo/file_record""github.com/apache/answer/internal/repo/limit""github.com/apache/answer/internal/repo/meta"notification2 "github.com/apache/answer/internal/repo/notification"@@ -84,6 +85,7 @@ import ("github.com/apache/answer/internal/service/dashboard""github.com/apache/answer/internal/service/event_queue"export2 "github.com/apache/answer/internal/service/export"+ file_record2 "github.com/apache/answer/internal/service/file_record""github.com/apache/answer/internal/service/follow""github.com/apache/answer/internal/service/importer"meta2 "github.com/apache/answer/internal/service/meta"@@ -227,10 +229,10 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,contentRevisionService := content.NewRevisionService(revisionRepo, userCommon, questionCommon, answerService, objService, questionRepo, answerRepo, tagRepo, tagCommonService, notificationQueueService, activityQueueService, reportRepo, reviewService, reviewActivityRepo)revisionController := controller.NewRevisionController(contentRevisionService, rankService)rankController := controller.NewRankController(rankService)- badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)+ userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)notificationRepo := notification2.NewNotificationRepo(dataData)pluginUserConfigRepo := plugin_config.NewPluginUserConfigRepo(dataData)- userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)+ badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)userAdminService := user_admin.NewUserAdminService(userAdminRepo, userRoleRelService, authService, userCommon, userActiveActivityRepo, siteInfoCommonService, emailService, questionRepo, answerRepo, commentCommonRepo, userExternalLoginRepo, notificationRepo, pluginUserConfigRepo, badgeAwardRepo)userAdminController := controller_admin.NewUserAdminController(userAdminService)reasonRepo := reason.NewReasonRepo(configService)@@ -246,7 +248,9 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,notificationController := controller.NewNotificationController(notificationService, rankService)dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData)dashboardController := controller.NewDashboardController(dashboardService)- uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService)+ fileRecordRepo := file_record.NewFileRecordRepo(dataData)+ fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService)+ uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService)uploadController := controller.NewUploadController(uploaderService)activityActivityRepo := activity.NewActivityRepo(dataData, configService)activityCommon := activity_common2.NewActivityCommon(activityRepo, activityQueueService)@@ -287,7 +291,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,renderController := controller.NewRenderController()pluginAPIRouter := router.NewPluginAPIRouter(connectorController, userCenterController, captchaController, embedController, renderController)ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, shortIDMiddleware, templateRouter, pluginAPIRouter, uiConf)- scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService)+ scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService, fileRecordService, serviceConf)application := newApplication(serverConf, ginEngine, scheduledTaskManager)return application, func() {cleanup2()diff --git a/configs/config.yaml b/configs/config.yamlindex 52671010..69c4e303 100644--- a/configs/config.yaml+++ b/configs/config.yaml@@ -33,6 +33,9 @@ swaggerui:address: ':80'service_config:upload_path: "/data/uploads"+ clean_up_uploads: true+ clean_orphan_uploads_period_hours: 48+ purge_deleted_files_period_days: 30ui:public_url: '/'api_url: '/'diff --git a/go.mod b/go.modindex 7e207e6b..89c6876b 100644--- a/go.mod+++ b/go.mod@@ -60,6 +60,7 @@ require (golang.org/x/crypto v0.27.0golang.org/x/image v0.20.0golang.org/x/net v0.29.0+ golang.org/x/text v0.18.0gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23dfgopkg.in/yaml.v3 v3.0.1modernc.org/sqlite v1.33.0@@ -160,7 +161,6 @@ require (golang.org/x/arch v0.10.0 // indirectgolang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirectgolang.org/x/sys v0.25.0 // indirect- golang.org/x/text v0.18.0 // indirectgolang.org/x/tools v0.25.0 // indirectgoogle.golang.org/protobuf v1.34.2 // indirectgopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirectdiff --git a/internal/base/constant/upload.go b/internal/base/constant/upload.goindex c01c9279..d9001fde 100644--- a/internal/base/constant/upload.go+++ b/internal/base/constant/upload.go@@ -25,4 +25,5 @@ const (PostSubPath = "post"BrandingSubPath = "branding"FilesPostSubPath = "files/post"+ DeletedSubPath = "deleted")diff --git a/internal/base/cron/cron.go b/internal/base/cron/cron.goindex 2e7a92ab..98e59dec 100644--- a/internal/base/cron/cron.go+++ b/internal/base/cron/cron.go@@ -24,6 +24,8 @@ import ("fmt""github.com/apache/answer/internal/service/content"+ "github.com/apache/answer/internal/service/file_record"+ "github.com/apache/answer/internal/service/service_config""github.com/apache/answer/internal/service/siteinfo_common""github.com/robfig/cron/v3""github.com/segmentfault/pacman/log"@@ -31,29 +33,36 @@ import (// ScheduledTaskManager scheduled task managertype ScheduledTaskManager struct {- siteInfoService siteinfo_common.SiteInfoCommonService- questionService *content.QuestionService+ siteInfoService siteinfo_common.SiteInfoCommonService+ questionService *content.QuestionService+ fileRecordService *file_record.FileRecordService+ serviceConfig *service_config.ServiceConfig}// NewScheduledTaskManager new scheduled task managerfunc NewScheduledTaskManager(siteInfoService siteinfo_common.SiteInfoCommonService,questionService *content.QuestionService,+ fileRecordService *file_record.FileRecordService,+ serviceConfig *service_config.ServiceConfig,) *ScheduledTaskManager {manager := &ScheduledTaskManager{- siteInfoService: siteInfoService,- questionService: questionService,+ siteInfoService: siteInfoService,+ questionService: questionService,+ fileRecordService: fileRecordService,+ serviceConfig: serviceConfig,}return manager}func (s *ScheduledTaskManager) Run() {- fmt.Println("start cron")+ log.Infof("cron job manager start")+s.questionService.SitemapCron(context.Background())c := cron.New()_, err := c.AddFunc("0 */1 * * *", func() {ctx := context.Background()- fmt.Println("sitemap cron execution")+ log.Infof("sitemap cron execution")s.questionService.SitemapCron(ctx)})if err != nil {@@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() {_, err = c.AddFunc("0 */1 * * *", func() {ctx := context.Background()- fmt.Println("refresh hottest cron execution")+ log.Infof("refresh hottest cron execution")s.questionService.RefreshHottestCron(ctx)})if err != nil {log.Error(err)}+ if s.serviceConfig.CleanUpUploads {+ log.Infof("clean up uploads cron enabled")++ conf := s.serviceConfig+ _, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", conf.CleanOrphanUploadsPeriodHours), func() {+ log.Infof("clean orphan upload files cron execution")+ s.fileRecordService.CleanOrphanUploadFiles(context.Background())+ })+ if err != nil {+ log.Error(err)+ }++ _, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", conf.PurgeDeletedFilesPeriodDays), func() {+ log.Infof("purge deleted files cron execution")+ s.fileRecordService.PurgeDeletedFiles(context.Background())+ })+ if err != nil {+ log.Error(err)+ }+ }c.Start()}diff --git a/internal/controller/upload_controller.go b/internal/controller/upload_controller.goindex 0bd74bf3..337cf83d 100644--- a/internal/controller/upload_controller.go+++ b/internal/controller/upload_controller.go@@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) {)source := ctx.PostForm("source")+ userID := middleware.GetLoginUserIDFromContext(ctx)switch source {case fileFromAvatar:- url, err = uc.uploaderService.UploadAvatarFile(ctx)+ url, err = uc.uploaderService.UploadAvatarFile(ctx, userID)case fileFromPost:- url, err = uc.uploaderService.UploadPostFile(ctx)+ url, err = uc.uploaderService.UploadPostFile(ctx, userID)case fileFromBranding:if !middleware.GetIsAdminFromContext(ctx) {handler.HandleResponse(ctx, errors.Forbidden(reason.ForbiddenError), nil)return}- url, err = uc.uploaderService.UploadBrandingFile(ctx)+ url, err = uc.uploaderService.UploadBrandingFile(ctx, userID)case fileFromPostAttachment:- url, err = uc.uploaderService.UploadPostAttachment(ctx)+ url, err = uc.uploaderService.UploadPostAttachment(ctx, userID)default:handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil)returndiff --git a/internal/entity/file_record_entity.go b/internal/entity/file_record_entity.gonew file mode 100644index 00000000..83a917aa--- /dev/null+++ b/internal/entity/file_record_entity.go@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package entity++import "time"++const (+ FileRecordStatusAvailable = 1+ FileRecordStatusDeleted = 10+)++// FileRecord file record+type FileRecord struct {+ ID int `xorm:"not null pk autoincr INT(10) id"`+ CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created TIMESTAMP created_at"`+ UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated TIMESTAMP updated_at"`+ UserID string `xorm:"not null default 0 BIGINT(20) user_id"`+ FilePath string `xorm:"not null VARCHAR(256) file_path"`+ FileURL string `xorm:"not null VARCHAR(1024) file_url"`+ ObjectID string `xorm:"not null default 0 INDEX BIGINT(20) object_id"`+ Source string `xorm:"not null VARCHAR(128) source"`+ Status int `xorm:"not null default 0 TINYINT(4) status"`+}++// TableName file record table name+func (FileRecord) TableName() string {+ return "file_record"+}diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.goindex bc2ecda7..57f4778e 100644--- a/internal/migrations/migrations.go+++ b/internal/migrations/migrations.go@@ -100,6 +100,7 @@ var migrations = []Migration{NewMigration("v1.4.0", "add badge/badge_group/badge_award table", addBadges, true),NewMigration("v1.4.1", "add question link", addQuestionLink, true),NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true),+ NewMigration("v1.4.5", "add file record", addFileRecord, true),}func GetMigrations() []Migration {diff --git a/internal/migrations/v25.go b/internal/migrations/v25.gonew file mode 100644index 00000000..a0e1f1c2--- /dev/null+++ b/internal/migrations/v25.go@@ -0,0 +1,31 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package migrations++import (+ "context"++ "github.com/apache/answer/internal/entity"+ "xorm.io/xorm"+)++func addFileRecord(ctx context.Context, x *xorm.Engine) error {+ return x.Context(ctx).Sync(new(entity.FileRecord))+}diff --git a/internal/repo/answer/answer_repo.go b/internal/repo/answer/answer_repo.goindex 2c2787f6..cdfc5802 100644--- a/internal/repo/answer/answer_repo.go+++ b/internal/repo/answer/answer_repo.go@@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, answerID string) (err er}func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error {- _, err := ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})+ // get all deleted answers ids+ ids := make([]string, 0)+ err := ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()).+ Where("status = ?", entity.AnswerStatusDeleted).Find(&ids)+ if err != nil {+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ if len(ids) == 0 {+ return nil+ }++ // delete all revisions permanently+ _, err = ar.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})+ if err != nil {+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }++ _, err = ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})if err != nil {return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()}diff --git a/internal/repo/file_record/file_record_repo.go b/internal/repo/file_record/file_record_repo.gonew file mode 100644index 00000000..ed081be4--- /dev/null+++ b/internal/repo/file_record/file_record_repo.go@@ -0,0 +1,84 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package file_record++import (+ "context"++ "github.com/apache/answer/internal/base/pager"+ "github.com/apache/answer/internal/service/file_record"++ "github.com/apache/answer/internal/base/data"+ "github.com/apache/answer/internal/base/reason"+ "github.com/apache/answer/internal/entity"+ "github.com/segmentfault/pacman/errors"+)++// fileRecordRepo fileRecord repository+type fileRecordRepo struct {+ data *data.Data+}++// NewFileRecordRepo new repository+func NewFileRecordRepo(data *data.Data) file_record.FileRecordRepo {+ return &fileRecordRepo{+ data: data,+ }+}++// AddFileRecord add file record+func (fr *fileRecordRepo) AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {+ _, err = fr.data.DB.Context(ctx).Insert(fileRecord)+ if err != nil {+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ return+}++// GetFileRecordPage get fileRecord page+func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (+ fileRecordList []*entity.FileRecord, total int64, err error) {+ fileRecordList = make([]*entity.FileRecord, 0)++ session := fr.data.DB.Context(ctx)+ total, err = pager.Help(page, pageSize, &fileRecordList, cond, session)+ if err != nil {+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ return+}++// DeleteFileRecord delete file record+func (fr *fileRecordRepo) DeleteFileRecord(ctx context.Context, id int) (err error) {+ _, err = fr.data.DB.Context(ctx).ID(id).Cols("status").Update(&entity.FileRecord{Status: entity.FileRecordStatusDeleted})+ if err != nil {+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ return+}++// UpdateFileRecord update file record+func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {+ _, err = fr.data.DB.Context(ctx).ID(fileRecord.ID).Update(fileRecord)+ if err != nil {+ err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ return+}diff --git a/internal/repo/provider.go b/internal/repo/provider.goindex 1b29cc5e..02f27f62 100644--- a/internal/repo/provider.go+++ b/internal/repo/provider.go@@ -33,6 +33,7 @@ import ("github.com/apache/answer/internal/repo/comment""github.com/apache/answer/internal/repo/config""github.com/apache/answer/internal/repo/export"+ "github.com/apache/answer/internal/repo/file_record""github.com/apache/answer/internal/repo/limit""github.com/apache/answer/internal/repo/meta""github.com/apache/answer/internal/repo/notification"@@ -107,4 +108,5 @@ var ProviderSetRepo = wire.NewSet(badge.NewEventRuleRepo,badge_group.NewBadgeGroupRepo,badge_award.NewBadgeAwardRepo,+ file_record.NewFileRecordRepo,)diff --git a/internal/repo/question/question_repo.go b/internal/repo/question/question_repo.goindex 117ee38f..978c9ce3 100644--- a/internal/repo/question/question_repo.go+++ b/internal/repo/question/question_repo.go@@ -168,6 +168,23 @@ func (qr *questionRepo) UpdateQuestionStatusWithOutUpdateTime(ctx context.Contex}func (qr *questionRepo) DeletePermanentlyQuestions(ctx context.Context) (err error) {+ // get all deleted question ids+ ids := make([]string, 0)+ err = qr.data.DB.Context(ctx).Select("id").Table(new(entity.Question).TableName()).+ Where("status = ?", entity.QuestionStatusDeleted).Find(&ids)+ if err != nil {+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ if len(ids) == 0 {+ return nil+ }++ // delete all revisions permanently+ _, err = qr.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})+ if err != nil {+ return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+_, err = qr.data.DB.Context(ctx).Where("status = ?", entity.QuestionStatusDeleted).Delete(&entity.Question{})if err != nil {return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()diff --git a/internal/repo/revision/revision_repo.go b/internal/repo/revision/revision_repo.goindex ab00adf9..09ba1aac 100644--- a/internal/repo/revision/revision_repo.go+++ b/internal/repo/revision/revision_repo.go@@ -155,7 +155,17 @@ func (rr *revisionRepo) GetLastRevisionByObjectID(ctx context.Context, objectIDrevision *entity.Revision, exist bool, err error,) {revision = &entity.Revision{}- exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).OrderBy("created_at DESC").Get(revision)+ exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).Desc("created_at").Get(revision)+ if err != nil {+ return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ }+ return+}++// GetLastRevisionByFileURL get object's last revision by file url+func (rr *revisionRepo) GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error) {+ revision = &entity.Revision{}+ exist, err = rr.data.DB.Context(ctx).Where("content LIKE ?", "%"+fileURL+"%").Desc("created_at").Get(revision)if err != nil {return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()}diff --git a/internal/router/static_router.go b/internal/router/static_router.goindex 807f4c73..a6c80fc0 100644--- a/internal/router/static_router.go+++ b/internal/router/static_router.go@@ -20,11 +20,14 @@package routerimport (+ "net/http"+ "path/filepath"+ "strings"+"github.com/apache/answer/internal/base/constant""github.com/apache/answer/internal/service/service_config"+ "github.com/apache/answer/pkg/dir""github.com/gin-gonic/gin"- "path/filepath"- "strings")// StaticRouter static api router@@ -54,6 +57,11 @@ func (a *StaticRouter) RegisterStaticRouter(r *gin.RouterGroup) {realFilename := strings.TrimSuffix(filePath, "/"+originalFilename) + filepath.Ext(originalFilename)// The file local path is /uploads/files/post/hash.pdffileLocalPath := filepath.Join(a.serviceConfig.UploadPath, constant.FilesPostSubPath, realFilename)+ // If the file is not exist, return 404+ if !dir.CheckFileExist(fileLocalPath) {+ c.Redirect(http.StatusFound, "/404")+ return+ }c.FileAttachment(fileLocalPath, originalFilename)})}diff --git a/internal/service/content/question_service.go b/internal/service/content/question_service.goindex 109b6b10..f3641dcc 100644--- a/internal/service/content/question_service.go+++ b/internal/service/content/question_service.go@@ -1398,6 +1398,8 @@ func (qs *QuestionService) GetQuestionPage(ctx context.Context, req *schema.Quesreturn nil, 0, err}tagIDs = append(synTagIds, tagInfo.ID)+ } else {+ return questions, 0, nil}}diff --git a/internal/service/file_record/file_record_service.go b/internal/service/file_record/file_record_service.gonew file mode 100644index 00000000..abb98376--- /dev/null+++ b/internal/service/file_record/file_record_service.go@@ -0,0 +1,172 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package file_record++import (+ "context"+ "fmt"+ "os"+ "path/filepath"+ "time"++ "github.com/apache/answer/internal/base/constant"+ "github.com/apache/answer/internal/entity"+ "github.com/apache/answer/internal/service/revision"+ "github.com/apache/answer/internal/service/service_config"+ "github.com/apache/answer/internal/service/siteinfo_common"+ "github.com/apache/answer/pkg/checker"+ "github.com/apache/answer/pkg/dir"+ "github.com/apache/answer/pkg/writer"+ "github.com/segmentfault/pacman/log"+)++// FileRecordRepo file record repository+type FileRecordRepo interface {+ AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)+ UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)+ GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (+ fileRecordList []*entity.FileRecord, total int64, err error)+ DeleteFileRecord(ctx context.Context, id int) (err error)+}++// FileRecordService file record service+type FileRecordService struct {+ fileRecordRepo FileRecordRepo+ revisionRepo revision.RevisionRepo+ serviceConfig *service_config.ServiceConfig+ siteInfoService siteinfo_common.SiteInfoCommonService+}++// NewFileRecordService new file record service+func NewFileRecordService(+ fileRecordRepo FileRecordRepo,+ revisionRepo revision.RevisionRepo,+ serviceConfig *service_config.ServiceConfig,+ siteInfoService siteinfo_common.SiteInfoCommonService,+) *FileRecordService {+ return &FileRecordService{+ fileRecordRepo: fileRecordRepo,+ revisionRepo: revisionRepo,+ serviceConfig: serviceConfig,+ siteInfoService: siteInfoService,+ }+}++// AddFileRecord add file record+func (fs *FileRecordService) AddFileRecord(ctx context.Context, userID, filePath, fileURL, source string) {+ record := &entity.FileRecord{+ UserID: userID,+ FilePath: filePath,+ FileURL: fileURL,+ Source: source,+ Status: entity.FileRecordStatusAvailable,+ ObjectID: "0",+ }+ if err := fs.fileRecordRepo.AddFileRecord(ctx, record); err != nil {+ log.Errorf("add file record error: %v", err)+ }+}++// CleanOrphanUploadFiles clean orphan upload files+func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) {+ page, pageSize := 1, 1000++ for {+ fileRecordList, total, err := fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{+ Status: entity.FileRecordStatusAvailable,+ })+ if err != nil {+ log.Errorf("get file record page error: %v", err)+ return+ }+ if len(fileRecordList) == 0 || total == 0 {+ break+ }+ for _, fileRecord := range fileRecordList {+ // If this file record created in 48 hours, no need to check+ if fileRecord.CreatedAt.AddDate(0, 0, 2).After(time.Now()) {+ continue+ }+ if checker.IsNotZeroString(fileRecord.ObjectID) {+ _, exist, err := fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID)+ if err != nil {+ log.Errorf("get last revision by object id error: %v", err)+ continue+ }+ if exist {+ continue+ }+ } else {+ lastRevision, exist, err := fs.revisionRepo.GetLastRevisionByFileURL(ctx, fileRecord.FileURL)+ if err != nil {+ log.Errorf("get last revision by file url error: %v", err)+ continue+ }+ if exist {+ // update the file record object id+ fileRecord.ObjectID = lastRevision.ObjectID+ if err := fs.fileRecordRepo.UpdateFileRecord(ctx, fileRecord); err != nil {+ log.Errorf("update file record object id error: %v", err)+ }+ continue+ }+ }+ // Delete and move the file record+ if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); err != nil {+ log.Error(err)+ }+ }+ page+++ }+}++func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) {+ deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath)+ log.Infof("purge deleted files: %s", deletedPath)+ err := os.RemoveAll(deletedPath)+ if err != nil {+ log.Errorf("purge deleted files error: %v", err)+ return+ }+ err = dir.CreateDirIfNotExist(deletedPath)+ if err != nil {+ log.Errorf("create deleted directory error: %v", err)+ }+ return+}++func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error {+ // Delete the file record+ if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err != nil {+ return fmt.Errorf("delete file record error: %v", err)+ }++ // Move the file to the deleted directory+ oldFilename := filepath.Base(fileRecord.FilePath)+ oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, fileRecord.FilePath)+ deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath, oldFilename)++ if err := writer.MoveFile(oldFilePath, deletedPath); err != nil {+ return fmt.Errorf("move file error: %v", err)+ }++ log.Debugf("delete and move file: %s", fileRecord.FileURL)+ return nil+}diff --git a/internal/service/object_info/object_info.go b/internal/service/object_info/object_info.goindex 76b1d86d..5ef438ff 100644--- a/internal/service/object_info/object_info.go+++ b/internal/service/object_info/object_info.go@@ -21,6 +21,7 @@ package object_infoimport ("context"+"github.com/apache/answer/internal/base/constant""github.com/apache/answer/internal/base/reason""github.com/apache/answer/internal/schema"@@ -279,7 +280,7 @@ func (os *ObjService) GetInfo(ctx context.Context, objectID string) (objInfo *scObjectID: tagInfo.ID,TagID: tagInfo.ID,ObjectType: objectType,- Title: tagInfo.ParsedText,+ Title: tagInfo.SlugName,Content: tagInfo.ParsedText, // todo trim}}diff --git a/internal/service/provider.go b/internal/service/provider.goindex 84073235..4b1b6427 100644--- a/internal/service/provider.go+++ b/internal/service/provider.go@@ -36,6 +36,7 @@ import ("github.com/apache/answer/internal/service/dashboard""github.com/apache/answer/internal/service/event_queue""github.com/apache/answer/internal/service/export"+ "github.com/apache/answer/internal/service/file_record""github.com/apache/answer/internal/service/follow""github.com/apache/answer/internal/service/importer""github.com/apache/answer/internal/service/meta"@@ -126,4 +127,5 @@ var ProviderSetService = wire.NewSet(badge.NewBadgeAwardService,badge.NewBadgeGroupService,importer.NewImporterService,+ file_record.NewFileRecordService,)diff --git a/internal/service/revision/revision.go b/internal/service/revision/revision.goindex 58427220..33ce285e 100644--- a/internal/service/revision/revision.go+++ b/internal/service/revision/revision.go@@ -31,6 +31,7 @@ type RevisionRepo interface {AddRevision(ctx context.Context, revision *entity.Revision, autoUpdateRevisionID bool) (err error)GetRevisionByID(ctx context.Context, revisionID string) (revision *entity.Revision, exist bool, err error)GetLastRevisionByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)+ GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error)GetRevisionList(ctx context.Context, revision *entity.Revision) (revisionList []entity.Revision, err error)UpdateObjectRevisionId(ctx context.Context, revision *entity.Revision, session *xorm.Session) (err error)ExistUnreviewedByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)diff --git a/internal/service/service_config/service_config.go b/internal/service/service_config/service_config.goindex c5f27669..90e399e1 100644--- a/internal/service/service_config/service_config.go+++ b/internal/service/service_config/service_config.go@@ -20,5 +20,8 @@package service_configtype ServiceConfig struct {- UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`+ UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`+ CleanUpUploads bool `json:"clean_up_uploads" mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"`+ CleanOrphanUploadsPeriodHours int `json:"clean_orphan_uploads_period_hours" mapstructure:"clean_orphan_uploads_period_hours" yaml:"clean_orphan_uploads_period_hours"`+ PurgeDeletedFilesPeriodDays int `json:"purge_deleted_files_period_days" mapstructure:"purge_deleted_files_period_days" yaml:"purge_deleted_files_period_days"`}diff --git a/internal/service/tag_common/tag_common.go b/internal/service/tag_common/tag_common.goindex e5c136c3..4ce89975 100644--- a/internal/service/tag_common/tag_common.go+++ b/internal/service/tag_common/tag_common.go@@ -349,10 +349,17 @@ func (ts *TagCommonService) AddTag(ctx context.Context, req *schema.AddTagReq) (}tagInfoJson, _ := json.Marshal(tagInfo)revisionDTO.Content = string(tagInfoJson)- _, err = ts.revisionService.AddRevision(ctx, revisionDTO, true)+ revisionID, err := ts.revisionService.AddRevision(ctx, revisionDTO, true)if err != nil {return nil, err}+ ts.activityQueueService.Send(ctx, &schema.ActivityMsg{+ UserID: req.UserID,+ ObjectID: tagInfo.ID,+ OriginalObjectID: tagInfo.ID,+ ActivityTypeKey: constant.ActTagCreated,+ RevisionID: revisionID,+ })return &schema.AddTagResp{SlugName: tagInfo.SlugName}, nil}diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.goindex 4b018cb4..d5cc2dfb 100644--- a/internal/service/uploader/upload.go+++ b/internal/service/uploader/upload.go@@ -31,6 +31,8 @@ import ("path/filepath""strings"+ "github.com/apache/answer/internal/service/file_record"+"github.com/apache/answer/internal/base/constant""github.com/apache/answer/internal/base/reason""github.com/apache/answer/internal/service/service_config"@@ -53,6 +55,7 @@ var (constant.PostSubPath,constant.BrandingSubPath,constant.FilesPostSubPath,+ constant.DeletedSubPath,}supportedThumbFileExtMapping = map[string]imaging.Format{".jpg": imaging.JPEG,@@ -63,22 +66,26 @@ var ()type UploaderService interface {- UploadAvatarFile(ctx *gin.Context) (url string, err error)- UploadPostFile(ctx *gin.Context) (url string, err error)- UploadPostAttachment(ctx *gin.Context) (url string, err error)- UploadBrandingFile(ctx *gin.Context) (url string, err error)+ UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error)+ UploadPostFile(ctx *gin.Context, userID string) (url string, err error)+ UploadPostAttachment(ctx *gin.Context, userID string) (url string, err error)+ UploadBrandingFile(ctx *gin.Context, userID string) (url string, err error)AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error)}// uploaderService uploader servicetype uploaderService struct {- serviceConfig *service_config.ServiceConfig- siteInfoService siteinfo_common.SiteInfoCommonService+ serviceConfig *service_config.ServiceConfig+ siteInfoService siteinfo_common.SiteInfoCommonService+ fileRecordService *file_record.FileRecordService}// NewUploaderService new upload service-func NewUploaderService(serviceConfig *service_config.ServiceConfig,- siteInfoService siteinfo_common.SiteInfoCommonService) UploaderService {+func NewUploaderService(+ serviceConfig *service_config.ServiceConfig,+ siteInfoService siteinfo_common.SiteInfoCommonService,+ fileRecordService *file_record.FileRecordService,+) UploaderService {for _, subPath := range subPathList {err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath))if err != nil {@@ -86,13 +93,14 @@ func NewUploaderService(serviceConfig *service_config.ServiceConfig,}}return &uploaderService{- serviceConfig: serviceConfig,- siteInfoService: siteInfoService,+ serviceConfig: serviceConfig,+ siteInfoService: siteInfoService,+ fileRecordService: fileRecordService,}}// UploadAvatarFile upload avatar file-func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err error) {+func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) {url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar)if err != nil {return "", err@@ -174,7 +182,7 @@ func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, sireturn saveFilePath, nil}-func (us *uploaderService) UploadPostFile(ctx *gin.Context) (+func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) (url string, err error) {url, err = us.tryToUploadByPlugin(ctx, plugin.UserPost)if err != nil {@@ -202,10 +210,15 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context) (fileExt := strings.ToLower(path.Ext(fileHeader.Filename))newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)avatarFilePath := path.Join(constant.PostSubPath, newFilename)- return us.uploadImageFile(ctx, fileHeader, avatarFilePath)+ url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath)+ if err != nil {+ return "", err+ }+ us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost))+ return url, nil}-func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (+func (us *uploaderService) UploadPostAttachment(ctx *gin.Context, userID string) (url string, err error) {url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment)if err != nil {@@ -232,11 +245,16 @@ func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (fileExt := strings.ToLower(path.Ext(fileHeader.Filename))newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)- avatarFilePath := path.Join(constant.FilesPostSubPath, newFilename)- return us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, avatarFilePath)+ attachmentFilePath := path.Join(constant.FilesPostSubPath, newFilename)+ url, err = us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, attachmentFilePath)+ if err != nil {+ return "", err+ }+ us.fileRecordService.AddFileRecord(ctx, userID, attachmentFilePath, url, string(plugin.UserPostAttachment))+ return url, nil}-func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) (+func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) (url string, err error) {url, err = us.tryToUploadByPlugin(ctx, plugin.AdminBranding)if err != nil {diff --git a/pkg/writer/writer.go b/pkg/writer/writer.goindex 49ded3df..ff08d259 100644--- a/pkg/writer/writer.go+++ b/pkg/writer/writer.go@@ -48,3 +48,8 @@ func WriteFile(filePath, content string) error {}return nil}++// MoveFile move file to new path+func MoveFile(oldPath, newPath string) error {+ return os.Rename(oldPath, newPath)+}