Skip to content

A Q&A platform software for teams at any scales. Whether it's a community forum, help center, or knowledge management platform, you can always count on Apache Answer.

refactor(sitemap): save sitemap in cache

LinkinStarsa23080df0af4
diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go
index 78a75763..bc612ccd 100644
--- a/cmd/wire_gen.go
+++ b/cmd/wire_gen.go
@@ -222,7 +222,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
authUserMiddleware := middleware.NewAuthUserMiddleware(authService, siteInfoCommonService)
avatarMiddleware := middleware.NewAvatarMiddleware(serviceConf, uploaderService)
shortIDMiddleware := middleware.NewShortIDMiddleware(siteInfoCommonService)
- templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, dataData, siteInfoCommonService)
+ templateRenderController := templaterender.NewTemplateRenderController(questionService, userService, tagService, answerService, commentService, siteInfoCommonService, questionRepo)
templateController := controller.NewTemplateController(templateRenderController, siteInfoCommonService)
templateRouter := router.NewTemplateRouter(templateController, templateRenderController, siteInfoController)
connectorController := controller.NewConnectorController(siteInfoCommonService, emailService, userExternalLoginService)
diff --git a/internal/base/constant/cache_key.go b/internal/base/constant/cache_key.go
index 9e9f9949..c4788dac 100644
--- a/internal/base/constant/cache_key.go
+++ b/internal/base/constant/cache_key.go
@@ -16,4 +16,7 @@ const (
ConfigKEY2ContentCacheKeyPrefix = "answer:config:key:"
ConnectorUserExternalInfoCacheKey = "answer:connector:"
ConnectorUserExternalInfoCacheTime = 10 * time.Minute
+ SiteMapQuestionCacheKeyPrefix = "answer:sitemap:question:%d"
+ SiteMapQuestionCacheTime = time.Hour
+ SitemapMaxSize = 50000
)
diff --git a/internal/controller/template_render/controller.go b/internal/controller/template_render/controller.go
index 8a852132..51a0fdd3 100644
--- a/internal/controller/template_render/controller.go
+++ b/internal/controller/template_render/controller.go
@@ -1,9 +1,9 @@
package templaterender
import (
+ questioncommon "github.com/answerdev/answer/internal/service/question_common"
"math"
- "github.com/answerdev/answer/internal/base/data"
"github.com/answerdev/answer/internal/service/comment"
"github.com/answerdev/answer/internal/service/siteinfo_common"
"github.com/google/wire"
@@ -24,8 +24,8 @@ type TemplateRenderController struct {
tagService *tag.TagService
answerService *service.AnswerService
commentService *comment.CommentService
- data *data.Data
siteInfoService siteinfo_common.SiteInfoCommonService
+ questionRepo questioncommon.QuestionRepo
}
func NewTemplateRenderController(
@@ -34,9 +34,8 @@ func NewTemplateRenderController(
tagService *tag.TagService,
answerService *service.AnswerService,
commentService *comment.CommentService,
- data *data.Data,
siteInfoService siteinfo_common.SiteInfoCommonService,
-
+ questionRepo questioncommon.QuestionRepo,
) *TemplateRenderController {
return &TemplateRenderController{
questionService: questionService,
@@ -44,7 +43,7 @@ func NewTemplateRenderController(
tagService: tagService,
answerService: answerService,
commentService: commentService,
- data: data,
+ questionRepo: questionRepo,
siteInfoService: siteInfoService,
}
}
diff --git a/internal/controller/template_render/index.go b/internal/controller/template_render/index.go
deleted file mode 100644
index 362739ec..00000000
--- a/internal/controller/template_render/index.go
+++ /dev/null
@@ -1 +0,0 @@
-package templaterender
diff --git a/internal/controller/template_render/question.go b/internal/controller/template_render/question.go
index 4b6b023f..e8ae67f4 100644
--- a/internal/controller/template_render/question.go
+++ b/internal/controller/template_render/question.go
@@ -1,8 +1,6 @@
package templaterender
import (
- "encoding/json"
- "fmt"
"html/template"
"net/http"
@@ -32,48 +30,45 @@ func (t *TemplateRenderController) Sitemap(ctx *gin.Context) {
return
}
- sitemapInfo := &schema.SiteMapList{}
- infoStr, err := t.data.Cache.GetString(ctx, schema.SitemapCachekey)
+ questions, err := t.questionRepo.SitemapQuestions(ctx, 0, constant.SitemapMaxSize)
if err != nil {
- log.Errorf("get Cache failed: %s", err)
- return
- }
- hasTitle := false
- if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
- hasTitle = true
- }
- if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
- log.Errorf("get sitemap info failed: %s", err)
+ log.Errorf("get sitemap questions failed: %s", err)
return
}
- if len(sitemapInfo.QuestionIDs) > 0 {
- //question url list
- ctx.Header("Content-Type", "application/xml")
+ ctx.Header("Content-Type", "application/xml")
+ if len(questions) < constant.SitemapMaxSize {
ctx.HTML(
http.StatusOK, "sitemap.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
- "list": sitemapInfo.QuestionIDs,
- "general": general,
- "hastitle": hasTitle,
- },
- )
- } else {
- //question list page
- ctx.Header("Content-Type", "application/xml")
- ctx.HTML(
- http.StatusOK, "sitemap-list.xml", gin.H{
- "xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
- "page": sitemapInfo.MaxPageNum,
+ "list": questions,
"general": general,
+ "hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
+ siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
},
)
return
}
+
+ questionNum, err := t.questionRepo.GetQuestionCount(ctx)
+ if err != nil {
+ log.Error("GetQuestionCount error", err)
+ return
+ }
+ var pageList []int64
+ for page := int64(1); page*constant.SitemapMaxSize < questionNum; page++ {
+ pageList = append(pageList, page)
+ }
+ ctx.HTML(
+ http.StatusOK, "sitemap-list.xml", gin.H{
+ "xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
+ "page": pageList,
+ "general": general,
+ },
+ )
}
func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error {
- sitemapInfo := &schema.SiteMapPageList{}
general, err := t.siteInfoService.GetSiteGeneral(ctx)
if err != nil {
log.Error("get site general failed:", err)
@@ -84,28 +79,20 @@ func (t *TemplateRenderController) SitemapPage(ctx *gin.Context, page int) error
log.Error("get site GetSiteSeo failed:", err)
return err
}
- hasTitle := false
- if siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle || siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID {
- hasTitle = true
- }
- cachekey := fmt.Sprintf(schema.SitemapPageCachekey, page)
- infoStr, err := t.data.Cache.GetString(ctx, cachekey)
+ questions, err := t.questionRepo.SitemapQuestions(ctx, page, constant.SitemapMaxSize)
if err != nil {
- //If there is no cache, return directly.
- return nil
- }
- if err = json.Unmarshal([]byte(infoStr), sitemapInfo); err != nil {
- log.Errorf("get sitemap info failed: %s", err)
+ log.Errorf("get sitemap questions failed: %s", err)
return err
}
ctx.Header("Content-Type", "application/xml")
ctx.HTML(
http.StatusOK, "sitemap.xml", gin.H{
"xmlHeader": template.HTML(`<?xml version="1.0" encoding="UTF-8"?>`),
- "list": sitemapInfo.PageData,
+ "list": questions,
"general": general,
- "hastitle": hasTitle,
+ "hastitle": siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitle ||
+ siteInfo.PermaLink == constant.PermaLinkQuestionIDAndTitleByShortID,
},
)
return nil
diff --git a/internal/repo/question/question_repo.go b/internal/repo/question/question_repo.go
index c14f0cb7..18685dca 100644
--- a/internal/repo/question/question_repo.go
+++ b/internal/repo/question/question_repo.go
@@ -2,7 +2,9 @@ package question
import (
"context"
+ "encoding/json"
"fmt"
+ "github.com/segmentfault/pacman/log"
"strings"
"time"
"unicode"
@@ -222,13 +224,13 @@ func (qr *questionRepo) GetQuestionList(ctx context.Context, question *entity.Qu
}
func (qr *questionRepo) GetQuestionCount(ctx context.Context) (count int64, err error) {
- questionList := make([]*entity.Question, 0)
-
- count, err = qr.data.DB.Context(ctx).In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed}).FindAndCount(&questionList)
+ session := qr.data.DB.Context(ctx)
+ session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
+ count, err = session.Count(&entity.Question{Show: entity.QuestionShow})
if err != nil {
- return count, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+ return 0, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
- return
+ return count, nil
}
func (qr *questionRepo) GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error) {
@@ -249,40 +251,51 @@ func (qr *questionRepo) GetQuestionCountByIDs(ctx context.Context, ids []string)
return
}
-func (qr *questionRepo) GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error) {
+func (qr *questionRepo) SitemapQuestions(ctx context.Context, page, pageSize int) (
+ questionIDList []*schema.SiteMapQuestionInfo, err error) {
questionIDList = make([]*schema.SiteMapQuestionInfo, 0)
+
+ // try to get sitemap data from cache
+ cacheKey := fmt.Sprintf(constant.SiteMapQuestionCacheKeyPrefix, page)
+ cacheData, err := qr.data.Cache.GetString(ctx, cacheKey)
+ if err == nil && len(cacheKey) > 0 {
+ _ = json.Unmarshal([]byte(cacheData), &questionIDList)
+ return questionIDList, nil
+ }
+
+ // get sitemap data from db
rows := make([]*entity.Question, 0)
- if page > 0 {
- page = page - 1
- } else {
- page = 0
- }
- if pageSize == 0 {
- pageSize = constant.DefaultPageSize
- }
- offset := page * pageSize
- session := qr.data.DB.Context(ctx).Table("question")
- session = session.In("question.status", []int{entity.QuestionStatusAvailable, entity.QuestionStatusClosed})
- session.And("question.show = ?", entity.QuestionShow)
- session = session.Limit(pageSize, offset)
- session = session.OrderBy("question.created_at asc")
- err = session.Select("id,title,created_at,post_update_time").Find(&rows)
+ session := qr.data.DB.Context(ctx)
+ session.Select("id,title,created_at,post_update_time")
+ session.Where("`show` = ?", entity.QuestionShow)
+ session.Where("status = ? OR status = ?", entity.QuestionStatusAvailable, entity.QuestionStatusClosed)
+ session.Limit(pageSize, page*pageSize)
+ session.Asc("created_at")
+ err = session.Find(&rows)
if err != nil {
return questionIDList, err
}
+
+ // warp data
for _, question := range rows {
item := &schema.SiteMapQuestionInfo{}
if handler.GetEnableShortID(ctx) {
item.ID = uid.EnShortID(question.ID)
}
item.Title = htmltext.UrlTitle(question.Title)
- updateTime := fmt.Sprintf("%v", question.PostUpdateTime.Format(time.RFC3339))
- if question.PostUpdateTime.Unix() < 1 {
- updateTime = fmt.Sprintf("%v", question.CreatedAt.Format(time.RFC3339))
+ if question.PostUpdateTime.IsZero() {
+ item.UpdateTime = question.CreatedAt.Format(time.RFC3339)
+ } else {
+ item.UpdateTime = question.PostUpdateTime.Format(time.RFC3339)
}
- item.UpdateTime = updateTime
questionIDList = append(questionIDList, item)
}
+
+ // set sitemap data to cache
+ cacheDataByte, _ := json.Marshal(questionIDList)
+ if err := qr.data.Cache.SetString(ctx, cacheKey, string(cacheDataByte), constant.SiteMapQuestionCacheTime); err != nil {
+ log.Error(err)
+ }
return questionIDList, nil
}
diff --git a/internal/router/template_router.go b/internal/router/template_router.go
index 4ecb99aa..e90dc7ee 100644
--- a/internal/router/template_router.go
+++ b/internal/router/template_router.go
@@ -17,7 +17,6 @@ func NewTemplateRouter(
templateController *controller.TemplateController,
templateRenderController *templaterender.TemplateRenderController,
siteInfoController *controller_admin.SiteInfoController,
-
) *TemplateRouter {
return &TemplateRouter{
templateController: templateController,
@@ -26,7 +25,7 @@ func NewTemplateRouter(
}
}
-// TemplateRouter template router
+// RegisterTemplateRouter template router
func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
r.GET("/sitemap.xml", a.templateController.Sitemap)
r.GET("/sitemap/:page", a.templateController.SitemapPage)
@@ -35,7 +34,6 @@ func (a *TemplateRouter) RegisterTemplateRouter(r *gin.RouterGroup) {
r.GET("/custom.css", a.siteInfoController.GetCss)
r.GET("/", a.templateController.Index)
- r.GET("/index", a.templateController.Index)
r.GET("/questions", a.templateController.QuestionList)
r.GET("/questions/:id", a.templateController.QuestionInfo)
diff --git a/internal/schema/question_schema.go b/internal/schema/question_schema.go
index 82c3f749..6fd26bab 100644
--- a/internal/schema/question_schema.go
+++ b/internal/schema/question_schema.go
@@ -11,9 +11,6 @@ import (
)
const (
- SitemapMaxSize = 50000
- SitemapCachekey = "answer@sitemap"
- SitemapPageCachekey = "answer@sitemap@page%d"
QuestionOperationPin = "pin"
QuestionOperationUnPin = "unpin"
QuestionOperationHide = "hide"
@@ -427,21 +424,6 @@ type AdminSetQuestionStatusRequest struct {
QuestionID string `json:"question_id" form:"question_id"`
}
-type SiteMapList struct {
- QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
- MaxPageNum []int `json:"max_page_num"`
-}
-
-type SiteMapPageList struct {
- PageData []*SiteMapQuestionInfo `json:"page_data"`
-}
-
-type SiteMapQuestionInfo struct {
- ID string `json:"id"`
- Title string `json:"title"`
- UpdateTime string `json:"time"`
-}
-
type PersonalQuestionPageReq struct {
Page int `validate:"omitempty,min=1" form:"page"`
PageSize int `validate:"omitempty,min=1" form:"page_size"`
diff --git a/internal/schema/sitemap_schema.go b/internal/schema/sitemap_schema.go
new file mode 100644
index 00000000..08e27f8d
--- /dev/null
+++ b/internal/schema/sitemap_schema.go
@@ -0,0 +1,16 @@
+package schema
+
+type SiteMapList struct {
+ QuestionIDs []*SiteMapQuestionInfo `json:"question_ids"`
+ MaxPageNum []int `json:"max_page_num"`
+}
+
+type SiteMapPageList struct {
+ PageData []*SiteMapQuestionInfo `json:"page_data"`
+}
+
+type SiteMapQuestionInfo struct {
+ ID string `json:"id"`
+ Title string `json:"title"`
+ UpdateTime string `json:"time"`
+}
diff --git a/internal/service/question_common/question.go b/internal/service/question_common/question.go
index 03d94752..ce93a947 100644
--- a/internal/service/question_common/question.go
+++ b/internal/service/question_common/question.go
@@ -3,7 +3,6 @@ package questioncommon
import (
"context"
"encoding/json"
- "fmt"
"math"
"time"
@@ -52,7 +51,7 @@ type QuestionRepo interface {
GetQuestionCount(ctx context.Context) (count int64, err error)
GetUserQuestionCount(ctx context.Context, userID string) (count int64, err error)
GetQuestionCountByIDs(ctx context.Context, ids []string) (count int64, err error)
- GetQuestionIDsPage(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
+ SitemapQuestions(ctx context.Context, page, pageSize int) (questionIDList []*schema.SiteMapQuestionInfo, err error)
}
// QuestionCommon user service
@@ -553,40 +552,26 @@ func (as *QuestionCommon) RemoveAnswer(ctx context.Context, id string) (err erro
}
func (qs *QuestionCommon) SitemapCron(ctx context.Context) {
- data := &schema.SiteMapList{}
questionNum, err := qs.questionRepo.GetQuestionCount(ctx)
if err != nil {
- log.Error("GetQuestionCount error", err)
+ log.Error(err)
return
}
- if questionNum <= schema.SitemapMaxSize {
- questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, 0, int(questionNum))
+ if questionNum <= constant.SitemapMaxSize {
+ _, err = qs.questionRepo.SitemapQuestions(ctx, 0, int(questionNum))
if err != nil {
- log.Error("GetQuestionIDsPage error", err)
- return
+ log.Errorf("get site map question error: %v", err)
}
- data.QuestionIDs = questionIDList
+ return
+ }
- } else {
- nums := make([]int, 0)
- totalpages := int(math.Ceil(float64(questionNum) / float64(schema.SitemapMaxSize)))
- for i := 1; i <= totalpages; i++ {
- siteMapPagedata := &schema.SiteMapPageList{}
- nums = append(nums, i)
- questionIDList, err := qs.questionRepo.GetQuestionIDsPage(ctx, i, int(schema.SitemapMaxSize))
- if err != nil {
- log.Error("GetQuestionIDsPage error", err)
- return
- }
- siteMapPagedata.PageData = questionIDList
- if setCacheErr := qs.SetCache(ctx, fmt.Sprintf(schema.SitemapPageCachekey, i), siteMapPagedata); setCacheErr != nil {
- log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
- }
+ totalPages := int(math.Ceil(float64(questionNum) / float64(constant.SitemapMaxSize)))
+ for i := 1; i <= totalPages; i++ {
+ _, err = qs.questionRepo.SitemapQuestions(ctx, i, constant.SitemapMaxSize)
+ if err != nil {
+ log.Errorf("get site map question error: %v", err)
+ return
}
- data.MaxPageNum = nums
- }
- if setCacheErr := qs.SetCache(ctx, schema.SitemapCachekey, data); setCacheErr != nil {
- log.Errorf("set sitemap cron SetCache failed: %s", setCacheErr)
}
}