BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr
Crawl other URL addresses in advance and add the crawling results to the knowledge base
hwliang6bb2b6cc8540
diff --git a/core/internal/controller/askai/askai_v1_project.go b/core/internal/controller/askai/askai_v1_project.goindex e0de3090..d144f8a3 100644--- a/core/internal/controller/askai/askai_v1_project.go+++ b/core/internal/controller/askai/askai_v1_project.go@@ -78,7 +78,7 @@ func (c *ControllerV1) ModifyKnowledgeBaseFile(ctx context.Context, req *v1.Modifunc (c *ControllerV1) CreateKnowledgeBaseFile(ctx context.Context, req *v1.CreateKnowledgeBaseFileReq) (res *v1.CreateKnowledgeBaseFileRes, err error) {res = &v1.CreateKnowledgeBaseFileRes{}- err = askai.CreateKnowledgeBaseFile(req.Domain, req.Title, req.Content)+ err = askai.CreateKnowledgeBaseFile(req.Domain, req.Title, req.Content, "")if err != nil {res.SetError(err)} else {diff --git a/core/internal/service/askai/openai.go b/core/internal/service/askai/openai.goindex 967a34ba..5c1b21de 100644--- a/core/internal/service/askai/openai.go+++ b/core/internal/service/askai/openai.go@@ -457,10 +457,10 @@ func (o *OpenAI) GetKnowledgeBasePrompt() (string, int) {return "", 0}- // 取内容长度小于1024的文档+ // 取内容长度小于10240的文档var filteredKnowledgeBase []stringfor i := len(knowledgeBase) - 1; i >= 0; i-- {- if len(knowledgeBase[i].Content) < 8192 {+ if len(knowledgeBase[i].Content) < 10240 {filteredKnowledgeBase = append(filteredKnowledgeBase, knowledgeBase[i].Content)}}diff --git a/core/internal/service/askai/project.go b/core/internal/service/askai/project.goindex 0b318408..7e6b8297 100644--- a/core/internal/service/askai/project.go+++ b/core/internal/service/askai/project.go@@ -350,8 +350,11 @@ func GetUUID() string {// CreateKnowledgeBaseFile creates a new knowledge base file with the provided domain, title, and content.// It generates a unique ID for the knowledge base, constructs a KnowledgeInfo struct, and saves it using the SaveKnowledgeBase function.-func CreateKnowledgeBaseFile(Domain string, Title string, Content string) error {- Tid := GetUUID()+func CreateKnowledgeBaseFile(Domain string, Title string, Content string, Tid string) error {+ if Tid == "" {+ Tid = GetUUID()+ }+knowledge := KnowledgeInfo{Kid: Tid,Title: Title,@@ -1026,8 +1029,150 @@ func GetImages(Domain string) ([]ImageInfo, error) {return images, nil}-func AutoGetProjectInfo() {+func AppendImages(domain string, Images []ImageInfo) {+ images, err := ReadImagesConfig(domain)+ if err != nil {+ fmt.Printf("Error reading images configuration for %s: %v\n", domain, err)+ return+ }++ for _, image := range Images {+ exists := false+ for _, img := range images {+ if img.ImageUrl == image.ImageUrl {+ exists = true+ break+ }+ }+ if !exists && image.ImageUrl != "" {+ image.ImageId = GetUUID()+ image.UpdateTime = public.GetNowTime()+ images = append(images, image)+ }+ }+ err = SaveImagesConfig(domain, images)+ if err != nil {+ fmt.Printf("Error saving images configuration for %s: %v\n", domain, err)+ return+ }+}++func AppendSitemap(domain string, Sitemap []BotSitemap) {+ siteMap, _ := GetSiteMap(domain)+ for _, sitemapNode := range Sitemap {+ // check if the node already exists in the site map+ exists := false+ for _, existingNode := range siteMap {+ if existingNode.UriPath == sitemapNode.URIPath {+ exists = true+ break+ }+ }+ if exists {+ continue+ }+ newNode := SiteMap{+ Title: sitemapNode.Title,+ // PageTitle: sitemapNode.PageTitle,+ // PageMarkdown: sitemapNode.PageMarkdown,+ UriPath: sitemapNode.URIPath,+ UpdateTime: public.GetNowTime(),+ }+ siteMap = append(siteMap, newNode)+ }+ SaveSiteMap(domain, siteMap)+}++func AppendStyle(domain string, Style BotStyle) {+ styleConfig, _ := GetStyleConfig(domain)+ styleConfig.AccentColor = Style.AccentColor+ styleConfig.TextColor = Style.TextColor+ styleConfig.PageBackground = Style.PageBackground+ styleConfig.ContainerBackground = Style.ContainerBackground+ styleConfig.LinkSocialColor = Style.LinkSocialColor+ styleConfig.LinkFooterColor = Style.LinkFooterColor+ styleConfig.HeadingFont = Style.HeadingFont+ styleConfig.BodyFont = Style.BodyFont+ styleConfig.UpdateTime = public.GetNowTime()+ SaveStyleConfig(domain, styleConfig)+}++func AppendCompanyConfig(domain string, Data BotData) {+ companyConfig, _ := GetCompanyProfile(domain)+ if companyConfig.LegalCompanyName == "" {+ companyConfig.LegalCompanyName = Data.Title+ }+ if companyConfig.WebSite == "" {+ companyConfig.WebSite = Data.URL+ }+ if companyConfig.CompanyProfile == "" {+ companyConfig.CompanyProfile = Data.Description+ }+ if companyConfig.Email == "" {+ companyConfig.Email = Data.Email+ }++ if companyConfig.Phone == "" {+ companyConfig.Phone = Data.Phone+ }+ if companyConfig.SupportUrl == "" {+ companyConfig.SupportUrl = Data.SupportUrl+ }+ companyConfig.UpdateTime = public.GetNowTime()+ SaveCompanyProfile(domain, companyConfig)+}++func AppendProjectConfig(domain string, config ProjectConfig, Data BotData) {+ config.Description = Data.Description+ config.Favicon = Data.Icon+ config.PrimaryLogo = Data.PrimaryLogo+ config.SecondaryLogo = Data.SecondaryLogo+ config.ProjectName = Data.Title+ config.UpdateTime = public.GetNowTime()+ SaveProjectConfig(domain, config)+}++func GetUrlIsRequest(domain string, urlMd5 string) bool {+ tipsPath := fmt.Sprintf("%s/%s/tips", PRODUCT_CONFIG_PATH, domain)+ if !public.FileExists(tipsPath) {+ os.MkdirAll(tipsPath, 0755)+ }++ tipsFile := fmt.Sprintf("%s/%s", tipsPath, urlMd5)+ return public.FileExists(tipsFile)+}++func SetUrlIsRequest(domain string, urlMd5 string) {+ tipsPath := fmt.Sprintf("%s/%s/tips", PRODUCT_CONFIG_PATH, domain)+ if !public.FileExists(tipsPath) {+ os.MkdirAll(tipsPath, 0755)+ }+ tipsFile := fmt.Sprintf("%s/%s", tipsPath, urlMd5)+ os.WriteFile(tipsFile, []byte(public.GetNowTimeStr()), 0600)+}++func RequestUrl(domain, toUrl string) (BotResponse, error) {+ if !strings.HasPrefix(toUrl, "http") {+ toUrl = "http://" + toUrl+ }+ var result BotResponse+ url := FILE_CDN_API + "/bot?url=" + toUrl+ resultBody, err := public.HttpGetSrc(url, 360)+ if err != nil {+ fmt.Printf("Error fetching project data for %s: %v\n", domain, err)+ return result, err+ }++ // Parse the JSON response+ err = json.Unmarshal([]byte(resultBody), &result)+ if err != nil {+ fmt.Printf("Error parsing project data for %s: %v\n", domain, err)+ return result, err+ }+ return result, nil+}+func AutoGetProjectInfo() {dirs, err := os.ReadDir(PRODUCT_CONFIG_PATH)if err != nil {fmt.Println("Error reading project config directory:", err)@@ -1044,7 +1189,6 @@ func AutoGetProjectInfo() {if !public.FileExists(projectConfigFile) {continue // Skip if project.json does not exist for this domain}-// Read the project configurationconfig, err := ReadProjectConfig(domain)if err != nil {@@ -1052,9 +1196,9 @@ func AutoGetProjectInfo() {continue}- if config.UpdateTime > 0 {- continue // Skip if the project configuration has already been updated- }+ // if config.UpdateTime > 0 {+ // continue // Skip if the project configuration has already been updated+ // }if len(config.Urls) == 0 {continue // Skip if there are no URLs to process@@ -1066,133 +1210,46 @@ func AutoGetProjectInfo() {if isLock != nil && isLock.(bool) {continue}- public.SetCache(domain, true, 300) // Set a lock for 300 seconds-- toUrl := config.Urls[0]- if !strings.HasPrefix(toUrl, "http") {- toUrl = "http://" + toUrl- }- url := FILE_CDN_API + "/bot?url=" + toUrl- resultBody, err := public.HttpGetSrc(url, 360)- if err != nil {- fmt.Printf("Error fetching project data for %s: %v\n", domain, err)- continue- }-- // Parse the JSON response- var result BotResponse- err = json.Unmarshal([]byte(resultBody), &result)- if err != nil {- fmt.Printf("Error parsing project data for %s: %v\n", domain, err)- continue- }-- config.Description = result.Data.Description- config.Favicon = result.Data.Icon- config.PrimaryLogo = result.Data.PrimaryLogo- config.SecondaryLogo = result.Data.SecondaryLogo- config.ProjectName = result.Data.Title- config.UpdateTime = public.GetNowTime()- SaveProjectConfig(domain, config)-- // Clear the cache for this domain- public.RemoveCache(domain)-- companyConfig, _ := GetCompanyProfile(domain)- if companyConfig.LegalCompanyName == "" {- companyConfig.LegalCompanyName = result.Data.Title- }- if companyConfig.WebSite == "" {- companyConfig.WebSite = result.Data.URL- }- if companyConfig.CompanyProfile == "" {- companyConfig.CompanyProfile = result.Data.Description- }- if companyConfig.Email == "" {- companyConfig.Email = result.Data.Email- }-- if companyConfig.Phone == "" {- companyConfig.Phone = result.Data.Phone- }- if companyConfig.SupportUrl == "" {- companyConfig.SupportUrl = result.Data.SupportUrl- }-- companyConfig.UpdateTime = public.GetNowTime()-- SaveCompanyProfile(domain, companyConfig)-- styleConfig, _ := GetStyleConfig(domain)+ public.SetCache(domain, true, 600) // Set a lock for 300 seconds- styleConfig.AccentColor = result.Data.Style.AccentColor- styleConfig.TextColor = result.Data.Style.TextColor- styleConfig.PageBackground = result.Data.Style.PageBackground- styleConfig.ContainerBackground = result.Data.Style.ContainerBackground- styleConfig.LinkSocialColor = result.Data.Style.LinkSocialColor- styleConfig.LinkFooterColor = result.Data.Style.LinkFooterColor- styleConfig.HeadingFont = result.Data.Style.HeadingFont- styleConfig.BodyFont = result.Data.Style.BodyFont- styleConfig.UpdateTime = public.GetNowTime()+ for i, toUrl := range config.Urls {+ urlMd5 := public.Md5(toUrl)- SaveStyleConfig(domain, styleConfig)-- siteMap, _ := GetSiteMap(domain)-- for _, sitemapNode := range result.Data.Sitemap {- // check if the node already exists in the site map- exists := false- for _, existingNode := range siteMap {- if existingNode.UriPath == sitemapNode.URIPath {- exists = true- break- }+ // check is requestd+ if GetUrlIsRequest(domain, urlMd5) {+ continue}- if exists {++ result, err := RequestUrl(domain, toUrl)+ if err != nil {continue}- newNode := SiteMap{- Title: sitemapNode.Title,- // PageTitle: sitemapNode.PageTitle,- // PageMarkdown: sitemapNode.PageMarkdown,- UriPath: sitemapNode.URIPath,- UpdateTime: public.GetNowTime(),+ if i == 0 {+ AppendProjectConfig(domain, config, result.Data)+ AppendCompanyConfig(domain, result.Data)+ AppendStyle(domain, result.Data.Style)+ if result.Data.Footer.Copyright != "" {+ ModifyFooter(domain, result.Data.Footer.Copyright, result.Data.Footer.Disclaimer, result.Data.Footer.Text)+ }+ } else {+ for i, sitemapNode := range result.Data.Sitemap {+ if strings.HasPrefix(sitemapNode.URIPath, "http") {+ continue+ }+ result.Data.Sitemap[i].URIPath = strings.Replace(result.Data.SupportUrl+result.Data.Sitemap[i].URIPath, "//", "/", 1)+ }}- siteMap = append(siteMap, newNode)- }- SaveSiteMap(domain, siteMap)- CreateKnowledgeBaseFile(domain, result.Data.Title, result.Data.Markdown)+ AppendSitemap(domain, result.Data.Sitemap)+ CreateKnowledgeBaseFile(domain, result.Data.Title, result.Data.Markdown, urlMd5)+ AppendImages(domain, result.Data.Images)- images, err := ReadImagesConfig(domain)- if err != nil {- fmt.Printf("Error reading images configuration for %s: %v\n", domain, err)- continue- }-- for _, image := range result.Data.Images {- exists := false- for _, img := range images {- if img.ImageUrl == image.ImageUrl {- exists = true- break- }- }- if !exists && image.ImageUrl != "" {- image.ImageId = GetUUID()- image.UpdateTime = public.GetNowTime()- images = append(images, image)- }- }- err = SaveImagesConfig(domain, images)- if err != nil {- fmt.Printf("Error saving images configuration for %s: %v\n", domain, err)- continue+ // tip requestd+ SetUrlIsRequest(domain, urlMd5)}- if result.Data.Footer.Copyright != "" {- ModifyFooter(domain, result.Data.Footer.Copyright, result.Data.Footer.Disclaimer, result.Data.Footer.Text)- }+ // Clear the cache for this domain+ public.RemoveCache(domain)}}