Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
feat: get image blob in backend (#495)
boojack2d49e96a8a97
diff --git a/plugin/crawler/website.go b/plugin/http_getter/html_meta.gosimilarity index 83%rename from plugin/crawler/website.gorename to plugin/http_getter/html_meta.goindex e48b277b0..ca2d8a619 100644--- a/plugin/crawler/website.go+++ b/plugin/http_getter/html_meta.go@@ -1,9 +1,10 @@-package crawler+package getterimport (+ "fmt""io""net/http"- urlUtil "net/url"+ "net/url""golang.org/x/net/html""golang.org/x/net/html/atom"@@ -15,19 +16,26 @@ type HTMLMeta struct {Image string `json:"image"`}-func GetWebsiteMeta(url string) (*HTMLMeta, error) {- if _, err := urlUtil.Parse(url); err != nil {+func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {+ if _, err := url.Parse(urlStr); err != nil {return nil, err}- response, err := http.Get(url)+ response, err := http.Get(urlStr)if err != nil {return nil, err}defer response.Body.Close()- htmlMeta := extractHTMLMeta(response.Body)+ mediatype, err := getMediatype(response)+ if err != nil {+ return nil, err+ }+ if mediatype != "text/html" {+ return nil, fmt.Errorf("Wrong website mediatype")+ }+ htmlMeta := extractHTMLMeta(response.Body)return htmlMeta, nil}diff --git a/plugin/crawler/website_test.go b/plugin/http_getter/html_meta_test.gosimilarity index 76%rename from plugin/crawler/website_test.gorename to plugin/http_getter/html_meta_test.goindex 655e77733..0c0e60138 100644--- a/plugin/crawler/website_test.go+++ b/plugin/http_getter/html_meta_test.go@@ -1,4 +1,4 @@-package crawler+package getterimport ("testing"@@ -6,19 +6,19 @@ import ("github.com/stretchr/testify/require")-func TestGetWebsiteMeta(t *testing.T) {+func TestGetHTMLMeta(t *testing.T) {tests := []struct {- url string+ urlStr stringhtmlMeta HTMLMeta}{{- url: "https://baidu.com",+ urlStr: "https://baidu.com",htmlMeta: HTMLMeta{Title: "百度一下,你就知道",},},{- url: "https://www.bytebase.com/blog/sql-review-tool-for-devs",+ urlStr: "https://www.bytebase.com/blog/sql-review-tool-for-devs",htmlMeta: HTMLMeta{Title: "The SQL Review Tool for Developers",Description: "Reviewing SQL can be somewhat tedious, yet is essential to keep your database fleet reliable. At Bytebase, we are building a developer-first SQL review tool to empower the DevOps system.",@@ -27,7 +27,7 @@ func TestGetWebsiteMeta(t *testing.T) {},}for _, test := range tests {- metadata, err := GetWebsiteMeta(test.url)+ metadata, err := GetHTMLMeta(test.urlStr)require.NoError(t, err)require.Equal(t, test.htmlMeta, *metadata)}diff --git a/plugin/crawler/crawler.go b/plugin/http_getter/http_getter.gosimilarity index 51%rename from plugin/crawler/crawler.gorename to plugin/http_getter/http_getter.goindex 12752e56c..db171171a 100644--- a/plugin/crawler/crawler.go+++ b/plugin/http_getter/http_getter.go@@ -1,4 +1,4 @@-// crawler is using to get resources from url.+// getter is using to get resources from url.// * Get metadata for website;// * Get image blob to avoid CORS;-package crawler+package getterdiff --git a/plugin/http_getter/image.go b/plugin/http_getter/image.gonew file mode 100644index 000000000..1dbd3220a--- /dev/null+++ b/plugin/http_getter/image.go@@ -0,0 +1,45 @@+package getter++import (+ "fmt"+ "io"+ "net/http"+ "net/url"+ "strings"+)++type Image struct {+ Blob []byte+ Mediatype string+}++func GetImage(urlStr string) (*Image, error) {+ if _, err := url.Parse(urlStr); err != nil {+ return nil, err+ }++ response, err := http.Get(urlStr)+ if err != nil {+ return nil, err+ }+ defer response.Body.Close()++ mediatype, err := getMediatype(response)+ if err != nil {+ return nil, err+ }+ if !strings.HasPrefix(mediatype, "image/") {+ return nil, fmt.Errorf("Wrong image mediatype")+ }++ bodyBytes, err := io.ReadAll(response.Body)+ if err != nil {+ return nil, err+ }++ image := &Image{+ Blob: bodyBytes,+ Mediatype: mediatype,+ }+ return image, nil+}diff --git a/plugin/http_getter/image_test.go b/plugin/http_getter/image_test.gonew file mode 100644index 000000000..a81d30310--- /dev/null+++ b/plugin/http_getter/image_test.go@@ -0,0 +1,21 @@+package getter++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetImage(t *testing.T) {+ tests := []struct {+ urlStr string+ }{+ {+ urlStr: "https://star-history.com/bytebase.webp",+ },+ }+ for _, test := range tests {+ _, err := GetImage(test.urlStr)+ require.NoError(t, err)+ }+}diff --git a/plugin/http_getter/util.go b/plugin/http_getter/util.gonew file mode 100644index 000000000..c9125ee82--- /dev/null+++ b/plugin/http_getter/util.go@@ -0,0 +1,15 @@+package getter++import (+ "mime"+ "net/http"+)++func getMediatype(response *http.Response) (string, error) {+ contentType := response.Header.Get("content-type")+ mediatype, _, err := mime.ParseMediaType(contentType)+ if err != nil {+ return "", err+ }+ return mediatype, nil+}diff --git a/server/crawler.go b/server/crawler.godeleted file mode 100644index b6845b156..000000000--- a/server/crawler.go+++ /dev/null@@ -1,38 +0,0 @@-package server--import (- "encoding/json"- "fmt"- "net/http"-- "github.com/labstack/echo/v4"- "github.com/usememos/memos/plugin/crawler"- metric "github.com/usememos/memos/plugin/metrics"-)--func (s *Server) registerCrawlerPublicRoutes(g *echo.Group) {- g.GET("/crawler/website", func(c echo.Context) error {- ctx := c.Request().Context()- url := c.QueryParam("url")- if url == "" {- return echo.NewHTTPError(http.StatusBadRequest, "Missing website url")- }-- htmlMeta, err := crawler.GetWebsiteMeta(url)- if err != nil {- return echo.NewHTTPError(http.StatusNotAcceptable, fmt.Sprintf("Failed to get website meta with url: %s", url)).SetInternal(err)- }- s.Collector.Collect(ctx, &metric.Metric{- Name: "crawler used",- Labels: map[string]string{- "type": "website",- },- })-- c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8)- if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(htmlMeta)); err != nil {- return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode website HTML meta").SetInternal(err)- }- return nil- })-}diff --git a/server/http_getter.go b/server/http_getter.gonew file mode 100644index 000000000..65da8053c--- /dev/null+++ b/server/http_getter.go@@ -0,0 +1,70 @@+package server++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/url"++ "github.com/labstack/echo/v4"+ getter "github.com/usememos/memos/plugin/http_getter"+ metric "github.com/usememos/memos/plugin/metrics"+)++func (s *Server) registerCrawlerPublicRoutes(g *echo.Group) {+ g.GET("/get/httpmeta", func(c echo.Context) error {+ ctx := c.Request().Context()+ urlStr := c.QueryParam("url")+ if urlStr == "" {+ return echo.NewHTTPError(http.StatusBadRequest, "Missing website url")+ }+ if _, err := url.Parse(urlStr); err != nil {+ return echo.NewHTTPError(http.StatusBadRequest, "Wrong url").SetInternal(err)+ }++ htmlMeta, err := getter.GetHTMLMeta(urlStr)+ if err != nil {+ return echo.NewHTTPError(http.StatusNotAcceptable, fmt.Sprintf("Failed to get website meta with url: %s", urlStr)).SetInternal(err)+ }+ s.Collector.Collect(ctx, &metric.Metric{+ Name: "getter used",+ Labels: map[string]string{+ "type": "httpmeta",+ },+ })++ c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8)+ if err := json.NewEncoder(c.Response().Writer).Encode(composeResponse(htmlMeta)); err != nil {+ return echo.NewHTTPError(http.StatusInternalServerError, "Failed to encode website HTML meta").SetInternal(err)+ }+ return nil+ })+ g.GET("/get/image", func(c echo.Context) error {+ ctx := c.Request().Context()+ urlStr := c.QueryParam("url")+ if urlStr == "" {+ return echo.NewHTTPError(http.StatusBadRequest, "Missing image url")+ }+ if _, err := url.Parse(urlStr); err != nil {+ return echo.NewHTTPError(http.StatusBadRequest, "Wrong url").SetInternal(err)+ }++ image, err := getter.GetImage(urlStr)+ if err != nil {+ return echo.NewHTTPError(http.StatusNotAcceptable, fmt.Sprintf("Failed to get image url: %s", urlStr)).SetInternal(err)+ }+ s.Collector.Collect(ctx, &metric.Metric{+ Name: "getter used",+ Labels: map[string]string{+ "type": "image",+ },+ })++ c.Response().Writer.WriteHeader(http.StatusOK)+ c.Response().Writer.Header().Set("Content-Type", image.Mediatype)+ if _, err := c.Response().Writer.Write(image.Blob); err != nil {+ return echo.NewHTTPError(http.StatusInternalServerError, "Failed to write image blob").SetInternal(err)+ }+ return nil+ })+}diff --git a/server/resource.go b/server/resource.goindex 14dbab82a..b21462faa 100644--- a/server/resource.go+++ b/server/resource.go@@ -161,7 +161,6 @@ func (s *Server) registerResourceRoutes(g *echo.Group) {if _, err := c.Response().Writer.Write(resource.Blob); err != nil {return echo.NewHTTPError(http.StatusInternalServerError, "Failed to write resource blob").SetInternal(err)}-return nil})diff --git a/web/src/labs/marked/parser/Image.ts b/web/src/labs/marked/parser/Image.tsindex b366e0d95..3432e5e26 100644--- a/web/src/labs/marked/parser/Image.ts+++ b/web/src/labs/marked/parser/Image.ts@@ -8,7 +8,8 @@ const renderer = (rawStr: string): string => {return rawStr;}- return `<img class='img' src='${escape(matchResult[1])}' />`;+ // NOTE: Get image blob from backend to avoid CORS.+ return `<img class='img' src='/o/get/image?url=${escape(matchResult[1])}' />`;};export default {