A Q&A platform software for teams at any scales. Whether it's a community forum, help center, or knowledge management platform, you can always count on Apache Answer.
refactor(sitemap): save sitemap in cache
LinkinStarsa23080df0af4
diff --git a/cmd/wire_gen.go b/cmd/wire_gen.goindex 78a75763..bc612ccd 100644--- a/cmd/wire_gen.go+++ b/cmd/wire_gen.go@@ -222,7 +222,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,authUserMiddleware := middleware.NewAuthUserMiddleware(authService, siteInfoCommonService)avatarMiddleware := middleware.NewAvatarMiddleware(serviceConf, uploaderService)shortIDMiddleware := middleware.NewShortIDMiddleware(siteInfoCommonService)- templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, dataData, siteInfoCommonService)+ templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, siteInfoCommonService, questionRepo)templateController := controller.NewTemplateController(templateRenderController, siteInfoCommonService)templateRouter := router.NewTemplateRouter(templateController, templateRenderController, siteInfoController)connectorController := controller.NewConnectorController(siteInfoCommonService, emailService, userExternalLoginService)diff --git a/internal/base/constant/cache_key.go b/internal/base/constant/cache_key.goindex 9e9f9949..c4788dac 100644--- a/internal/base/constant/cache_key.go+++ b/internal/base/constant/cache_key.go@@ -16,4 +16,7 @@ const (ConfigKEY2ContentCacheKeyPrefix = "answer:config:key:"ConnectorUserExternalInfoCacheKey = "answer:connector:"ConnectorUserExternalInfoCacheTime = 10 * time.Minute+ SiteMapQuestionCacheKeyPrefix = "answer:sitemap:question:%d"+ SiteMapQuestionCacheTime = time.Hour+ SitemapMaxSize = 50000)diff --git a/internal/controller/template_render/controller.go b/internal/controller/template_render/controller.goindex 8a852132..51a0fdd3 100644--- a/internal/controller/template_render/controller.go+++ b/internal/controller/template_render/controller.go@@ -1,9 +1,9 @@package templaterenderimport (+ questioncommon "github.com/answerdev/answer/internal/service/question_common""math"- "github.com/answerdev/answer/internal/base/data""github.com/answerdev/answer/internal/service/comment""github.com/answerdev/answer/internal/service/siteinfo_common""github.com/google/wire"@@ -24,8 +24,8 @@ type TemplateRenderController struct {tagService *tag.TagServiceanswerService *service.AnswerServicecommentService *comment.CommentService- data *data.DatasiteInfoService siteinfo_common.SiteInfoCommonService+ questionRepo questioncommon.QuestionRepo}func NewTemplateRenderController(@@ -34,9 +34,8 @@ func NewTemplateRenderController(tagService *tag.TagService,answerService *service.AnswerService,commentService *comment.CommentService,- data *data.Data,siteInfoService siteinfo_common.SiteInfoCommonService,-+ questionRepo questioncommon.QuestionRepo,) *TemplateRenderController {return &TemplateRenderController{questionService: questionService,@@ -44,7 +43,7 @@ func NewTemplateRenderController(tagService: tagService,answerService: answerService,commentService: commentService,- data: data,+ questionRepo: questionRepo,siteInfoService: siteInfoService,}}diff --git a/internal/controller/template_render/index.go b/internal/controller/template_render/index.godeleted file mode 100644index 362739ec..00000000--- a/internal/controller/template_render/index.go+++ /dev/null@@ -1 +0,0 @@-package templaterenderdiff --git a/internal/controller/template_render/question.go b/internal/controller/template_render/question.goindex 4b6b023f..e8ae67f4 100644--- a/internal/controller/template_render/question.go+++ b/internal/controller/template_render/question.go@@ -1,8 +1,6 @@package templaterenderimport (- "encoding/json"- "fmt""html/template""net/http"@@ -32,48 +30,45 @@ func (t *TemplateRenderController) Sitemap(ctx *gin.Context) {return}- sitemapInfo := &schema.SiteMapList{}- infoStr, err := t.data.Cache.GetString(ctx, schema.SitemapCachekey)+ questions, err := t.questionRepo.SitemapQuestions(ctx, 0, constant.SitemapMaxSize)if err != nil {- log.Errorf("get Cache failed: %s", err)- return- }- hasTitle := false- if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {- hasTitle = true- }- if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {- log.Errorf("get sitemap info failed: %s", err)+ log.Errorf("get sitemap questions failed: %s", err)return}- if len(sitemapInfo.QuestionIDs) > 0 {- //question url list- ctx.Header("Content-Type", "application/xml")+ ctx.Header("Content-Type", "application/xml")+ if len(questions) < constant.SitemapMaxSize {ctx.HTML(http.StatusOK, "sitemap.xml", gin.H{"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),- "list": sitemapInfo.QuestionIDs,- "general": general,- "hastitle": hasTitle,- },- )- } else {- //question list page- ctx.Header("Content-Type", "application/xml")- ctx.HTML(- http.StatusOK, "sitemap-list.xml", gin.H{- "xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),- "page": sitemapInfo.MaxPageNum,+ "list": questions,"general": general,+ "hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||+ siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,},)return}++ questionNum, err := t.questionRepo.GetQuestionCount(ctx)+ if err != nil {+ log.Error("GetQuestionCount error", err)+ return+ }+ var pageList []int64+ for page := int64(1); page*constant.SitemapMaxSize < questionNum; page++ {+ pageList = append(pageList, page)+ }+ ctx.HTML(+ http.StatusOK, "sitemap-list.xml", gin.H{+ "xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),+ "page": pageList,+ "general": general,+ },+ )}func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error {- sitemapInfo := &schema.SiteMapPageList{}general, err := t.siteInfoService.GetSiteGeneral(ctx)if err != nil {log.Error("get site general failed:", err)@@ -84,28 +79,20 @@ func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) errorlog.Error("get site GetSiteSeo failed:", err)return err}- hasTitle := false- if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {- hasTitle = true- }- cachekey := fmt.Sprintf(schema.SitemapPageCachekey, page)- infoStr, err := t.data.Cache.GetString(ctx, cachekey)+ questions, err := t.questionRepo.SitemapQuestions(ctx, page, constant.SitemapMaxSize)if err != nil {- //If there is no cache, return directly.- return nil- }- if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {- log.Errorf("get sitemap info failed: %s", err)+ log.Errorf("get sitemap questions failed: %s", err)return err}ctx.Header("Content-Type", "application/xml")ctx.HTML(http.StatusOK, "sitemap.xml", gin.H{"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),- "list": sitemapInfo.PageData,+ "list": questions,"general": general,- "hastitle": hasTitle,+ "hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||+ siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,},)return nildiff --git a/internal/repo/question/question_repo.go b/internal/repo/question/question_repo.goindex c14f0cb7..18685dca 100644--- a/internal/repo/question/question_repo.go+++ b/internal/repo/question/question_repo.go@@ -2,7 +2,9 @@ package questionimport ("context"+ "encoding/json""fmt"+ "github.com/segmentfault/pacman/log""strings""time""unicode"@@ -222,13 +224,13 @@ func (qr *questionRepo) GetQuestionList(ctx context.Context, question *entity.Qu}func (qr *questionRepo) GetQuestionCount(ctx context.Context) (count int64, err error) {- questionList := make([]*entity.Question, 0)-- count, err = qr.data.DB.Context(ctx).In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed}).FindAndCount(&questionList)+ session := qr.data.DB.Context(ctx)+ session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)+ count, err = session.Count(&entity.Question{Show: entity.QuestionShow})if err != nil {- return count, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()+ return 0, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()}- return+ return count, nil}func (qr *questionRepo) GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error) {@@ -249,40 +251,51 @@ func (qr *questionRepo) GetQuestionCountByIDs(ctx context.Context, ids []string)return}-func (qr *questionRepo) GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error) {+func (qr *questionRepo) SitemapQuestions(ctx context.Context, page, pageSize int) (+ questionIDList []*schema.SiteMapQuestionInfo, err error) {questionIDList = make([]*schema.SiteMapQuestionInfo, 0)++ // try to get sitemap data from cache+ cacheKey := fmt.Sprintf(constant.SiteMapQuestionCacheKeyPrefix, page)+ cacheData, err := qr.data.Cache.GetString(ctx, cacheKey)+ if err == nil && len(cacheKey) > 0 {+ _ = json.Unmarshal([]byte(cacheData), &questionIDList)+ return questionIDList, nil+ }++ // get sitemap data from dbrows := make([]*entity.Question, 0)- if page > 0 {- page = page - 1- } else {- page = 0- }- if pageSize == 0 {- pageSize = constant.DefaultPageSize- }- offset := page * pageSize- session := qr.data.DB.Context(ctx).Table("question")- session = session.In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed})- session.And("question.show = ?", entity.QuestionShow)- session = session.Limit(pageSize, offset)- session = session.OrderBy("question.created_at asc")- err = session.Select("id,title,created_at,post_update_time").Find(&rows)+ session := qr.data.DB.Context(ctx)+ session.Select("id,title,created_at,post_update_time")+ session.Where("`show` = ?", entity.QuestionShow)+ session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)+ session.Limit(pageSize, page*pageSize)+ session.Asc("created_at")+ err = session.Find(&rows)if err != nil {return questionIDList, err}++ // warp datafor _, question := range rows {item := &schema.SiteMapQuestionInfo{}if handler.GetEnableShortID(ctx) {item.ID = uid.EnShortID(question.ID)}item.Title = htmltext.UrlTitle(question.Title)- updateTime := fmt.Sprintf("%v", question.PostUpdateTime.Format(time.RFC3339))- if question.PostUpdateTime.Unix() < 1 {- updateTime = fmt.Sprintf("%v", question.CreatedAt.Format(time.RFC3339))+ if question.PostUpdateTime.IsZero() {+ item.UpdateTime = question.CreatedAt.Format(time.RFC3339)+ } else {+ item.UpdateTime = question.PostUpdateTime.Format(time.RFC3339)}- item.UpdateTime = updateTimequestionIDList = append(questionIDList, item)}++ // set sitemap data to cache+ cacheDataByte, _ := json.Marshal(questionIDList)+ if err := qr.data.Cache.SetString(ctx, cacheKey, string(cacheDataByte), constant.SiteMapQuestionCacheTime); err != nil {+ log.Error(err)+ }return questionIDList, nil}diff --git a/internal/router/template_router.go b/internal/router/template_router.goindex 4ecb99aa..e90dc7ee 100644--- a/internal/router/template_router.go+++ b/internal/router/template_router.go@@ -17,7 +17,6 @@ func NewTemplateRouter(templateController *controller.TemplateController,templateRenderController *templaterender.TemplateRenderController,siteInfoController *controller_admin.SiteInfoController,-) *TemplateRouter {return &TemplateRouter{templateController: templateController,@@ -26,7 +25,7 @@ func NewTemplateRouter(}}-// TemplateRouter template router+// RegisterTemplateRouter template routerfunc (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {r.GET("/sitemap.xml", a.templateController.Sitemap)r.GET("/sitemap/:page", a.templateController.SitemapPage)@@ -35,7 +34,6 @@ func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {r.GET("/custom.css", a.siteInfoController.GetCss)r.GET("/", a.templateController.Index)- r.GET("/index", a.templateController.Index)r.GET("/questions", a.templateController.QuestionList)r.GET("/questions/:id", a.templateController.QuestionInfo)diff --git a/internal/schema/question_schema.go b/internal/schema/question_schema.goindex 82c3f749..6fd26bab 100644--- a/internal/schema/question_schema.go+++ b/internal/schema/question_schema.go@@ -11,9 +11,6 @@ import ()const (- SitemapMaxSize = 50000- SitemapCachekey = "answer@sitemap"- SitemapPageCachekey = "answer@sitemap@page%d"QuestionOperationPin = "pin"QuestionOperationUnPin = "unpin"QuestionOperationHide = "hide"@@ -427,21 +424,6 @@ type AdminSetQuestionStatusRequest struct {QuestionID string `json:"question_id" form:"question_id"`}-type SiteMapList struct {- QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`- MaxPageNum []int `json:"max_page_num"`-}--type SiteMapPageList struct {- PageData []*SiteMapQuestionInfo `json:"page_data"`-}--type SiteMapQuestionInfo struct {- ID string `json:"id"`- Title string `json:"title"`- UpdateTime string `json:"time"`-}-type PersonalQuestionPageReq struct {Page int `validate:"omitempty,min=1" form:"page"`PageSize int `validate:"omitempty,min=1" form:"page_size"`diff --git a/internal/schema/sitemap_schema.go b/internal/schema/sitemap_schema.gonew file mode 100644index 00000000..08e27f8d--- /dev/null+++ b/internal/schema/sitemap_schema.go@@ -0,0 +1,16 @@+package schema++type SiteMapList struct {+ QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`+ MaxPageNum []int `json:"max_page_num"`+}++type SiteMapPageList struct {+ PageData []*SiteMapQuestionInfo `json:"page_data"`+}++type SiteMapQuestionInfo struct {+ ID string `json:"id"`+ Title string `json:"title"`+ UpdateTime string `json:"time"`+}diff --git a/internal/service/question_common/question.go b/internal/service/question_common/question.goindex 03d94752..ce93a947 100644--- a/internal/service/question_common/question.go+++ b/internal/service/question_common/question.go@@ -3,7 +3,6 @@ package questioncommonimport ("context""encoding/json"- "fmt""math""time"@@ -52,7 +51,7 @@ type QuestionRepo interface {GetQuestionCount(ctx context.Context) (count int64, err error)GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error)GetQuestionCountByIDs(ctx context.Context, ids []string) (count int64, err error)- GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)+ SitemapQuestions(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)}// QuestionCommon user service@@ -553,40 +552,26 @@ func (as *QuestionCommon) RemoveAnswer(ctx context.Context, id string) (err erro}func (qs *QuestionCommon) SitemapCron(ctx context.Context) {- data := &schema.SiteMapList{}questionNum, err := qs.questionRepo.GetQuestionCount(ctx)if err != nil {- log.Error("GetQuestionCount error", err)+ log.Error(err)return}- if questionNum <= schema.SitemapMaxSize {- questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, 0, int(questionNum))+ if questionNum <= constant.SitemapMaxSize {+ _, err = qs.questionRepo.SitemapQuestions(ctx, 0, int(questionNum))if err != nil {- log.Error("GetQuestionIDsPage error", err)- return+ log.Errorf("get site map question error: %v", err)}- data.QuestionIDs = questionIDList+ return+ }- } else {- nums := make([]int, 0)- totalpages := int(math.Ceil(float64(questionNum) / float64(schema.SitemapMaxSize)))- for i := 1; i <= totalpages; i++ {- siteMapPagedata := &schema.SiteMapPageList{}- nums = append(nums, i)- questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, i, int(schema.SitemapMaxSize))- if err != nil {- log.Error("GetQuestionIDsPage error", err)- return- }- siteMapPagedata.PageData = questionIDList- if setCacheErr := qs.SetCache(ctx, fmt.Sprintf(schema.SitemapPageCachekey, i), siteMapPagedata); setCacheErr != nil {- log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)- }+ totalPages := int(math.Ceil(float64(questionNum) / float64(constant.SitemapMaxSize)))+ for i := 1; i <= totalPages; i++ {+ _, err = qs.questionRepo.SitemapQuestions(ctx, i, constant.SitemapMaxSize)+ if err != nil {+ log.Errorf("get site map question error: %v", err)+ return}- data.MaxPageNum = nums- }- if setCacheErr := qs.SetCache(ctx, schema.SitemapCachekey, data); setCacheErr != nil {- log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)}}