Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
fix(linkmeta): read further for missing preview metadata
johnnyjoygh0b3b5a699906
diff --git a/internal/linkmeta/html_meta.go b/internal/linkmeta/html_meta.goindex e2f2c2ad0..643e4e19e 100644--- a/internal/linkmeta/html_meta.go+++ b/internal/linkmeta/html_meta.go@@ -1,6 +1,7 @@package linkmetaimport (+ "bytes""context""io""net"@@ -22,7 +23,8 @@ var ErrInternalIP = errors.New("internal IP addresses are not allowed")const (defaultLinkPreviewUserAgent = "MemosBot/1.0 (+https://usememos.com)"- maxHTMLMetaBytes = 512 * 1024+ initialHTMLMetaBytes = 512 * 1024+ maxHTMLMetaBytes = 2 * 1024 * 1024maxOEmbedBytes = 128 * 1024maxCacheEntries = 1000maxConcurrentFetches = 8@@ -252,26 +254,53 @@ func (f *HTMLMetaFetcher) fetch(ctx context.Context, urlStr string) (*HTMLMeta,return nil, errors.New("not a HTML page")}- document, err := html.Parse(io.LimitReader(response.Body, maxHTMLMetaBytes))+ htmlBody, err := io.ReadAll(io.LimitReader(response.Body, initialHTMLMetaBytes))if err != nil {- return nil, errors.Wrap(err, "failed to parse HTML")+ return nil, errors.Wrap(err, "failed to read HTML")}pageURL, err := finalResponseURL(response, urlStr)if err != nil {return nil, err}- sources, oEmbedEndpoint := extractDocumentMetadata(document)- baseURL := resolveDocumentBase(pageURL, sources.baseHref)- var oEmbed metadataSource- if endpoint := resolveHTTPURL(baseURL, oEmbedEndpoint); endpoint != "" {- // oEmbed is optional enrichment, so its failure must not discard the- // metadata already extracted from the HTML document.- oEmbed, _ = f.fetchOEmbed(ctx, endpoint)+ var fetchedOEmbedEndpoint string+ var fetchedOEmbed metadataSource+ extract := func() (*HTMLMeta, error) {+ document, err := html.Parse(bytes.NewReader(htmlBody))+ if err != nil {+ return nil, errors.Wrap(err, "failed to parse HTML")+ }+ sources, oEmbedEndpoint := extractDocumentMetadata(document)+ baseURL := resolveDocumentBase(pageURL, sources.baseHref)+ var oEmbed metadataSource+ if endpoint := resolveHTTPURL(baseURL, oEmbedEndpoint); endpoint != "" {+ if endpoint != fetchedOEmbedEndpoint {+ // oEmbed is optional enrichment, so its failure must not discard+ // metadata already extracted from the HTML document.+ fetchedOEmbed, _ = f.fetchOEmbed(ctx, endpoint)+ fetchedOEmbedEndpoint = endpoint+ }+ oEmbed = fetchedOEmbed+ }+ return mergeMetadata(baseURL, oEmbed, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, sources.semantic), nil}- meta := mergeMetadata(baseURL, oEmbed, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, sources.semantic)- return meta, nil+ meta, err := extract()+ if err != nil || (meta.Title != "" && meta.Image != "") || len(htmlBody) < initialHTMLMetaBytes {+ return meta, err+ }++ // Metadata can follow a large inline script. Read further only when the+ // initial document lacks a title or image, while keeping a hard byte cap.+ rest, err := io.ReadAll(io.LimitReader(response.Body, maxHTMLMetaBytes-int64(len(htmlBody))))+ if err != nil {+ return nil, errors.Wrap(err, "failed to read HTML")+ }+ if len(rest) == 0 {+ return meta, nil+ }+ htmlBody = append(htmlBody, rest...)+ return extract()}func (f *HTMLMetaFetcher) fetchOEmbed(ctx context.Context, endpoint string) (metadataSource, error) {diff --git a/internal/linkmeta/html_meta_test.go b/internal/linkmeta/html_meta_test.goindex c1aadb3dd..6b4df7904 100644--- a/internal/linkmeta/html_meta_test.go+++ b/internal/linkmeta/html_meta_test.go@@ -291,7 +291,7 @@ func TestHTMLMetaFetcherValidatesResponse(t *testing.T) {}func TestHTMLMetaFetcherBoundsHTMLResponse(t *testing.T) {- body := `<title>Bounded title</title>` + strings.Repeat("x", maxHTMLMetaBytes) + `<meta property="og:title" content="Outside limit">`+ body := `<title>Bounded title</title><script>` + strings.Repeat("x", maxHTMLMetaBytes) + `</script><meta property="og:image" content="/outside.png">`reader := &countingReadCloser{Reader: strings.NewReader(body)}fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {result := response(req, http.StatusOK, "text/html", "")@@ -302,9 +302,48 @@ func TestHTMLMetaFetcherBoundsHTMLResponse(t *testing.T) {meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")require.NoError(t, err)require.Equal(t, "Bounded title", meta.Title)+ require.Empty(t, meta.Image)require.LessOrEqual(t, reader.bytesRead, int64(maxHTMLMetaBytes))}+func TestHTMLMetaFetcherStopsAfterInitialLimitWhenMetadataIsComplete(t *testing.T) {+ body := `<meta property="og:title" content="Early title"><meta property="og:image" content="/early.png"><script>` + strings.Repeat("x", maxHTMLMetaBytes)+ reader := &countingReadCloser{Reader: strings.NewReader(body)}+ fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {+ result := response(req, http.StatusOK, "text/html", "")+ result.Body = reader+ return result, nil+ }))++ meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")+ require.NoError(t, err)+ require.Equal(t, "Early title", meta.Title)+ require.Equal(t, "http://93.184.216.34/early.png", meta.Image)+ require.LessOrEqual(t, reader.bytesRead, int64(initialHTMLMetaBytes))+}++func TestHTMLMetaFetcherFindsOEmbedPastInitialLimit(t *testing.T) {+ body := `<html><head><script>` + strings.Repeat("x", initialHTMLMetaBytes) + `</script><link rel="alternate" type="application/json+oembed" href="/oembed"></head></html>`+ var requests atomic.Int32+ fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {+ requests.Add(1)+ switch req.URL.Path {+ case "/post":+ return htmlResponse(req, body), nil+ case "/oembed":+ return response(req, http.StatusOK, "application/json", `{"title":"Late title","thumbnail_url":"/late.png"}`), nil+ default:+ return nil, errors.New("unexpected request path: " + req.URL.Path)+ }+ }))++ meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/post")+ require.NoError(t, err)+ require.Equal(t, "Late title", meta.Title)+ require.Equal(t, "http://93.184.216.34/late.png", meta.Image)+ require.EqualValues(t, 2, requests.Load())+}+func TestHTMLMetaFetcherContextCancellation(t *testing.T) {requestStarted := make(chan struct{})fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {