Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

fix(linkmeta): read further for missing preview metadata

johnnyjoygh0b3b5a699906
diff --git a/internal/linkmeta/html_meta.go b/internal/linkmeta/html_meta.go
index e2f2c2ad0..643e4e19e 100644
--- a/internal/linkmeta/html_meta.go
+++ b/internal/linkmeta/html_meta.go
@@ -1,6 +1,7 @@
package linkmeta
import (
+ "bytes"
"context"
"io"
"net"
@@ -22,7 +23,8 @@ var ErrInternalIP = errors.New("internal IP addresses are not allowed")
const (
defaultLinkPreviewUserAgent = "MemosBot/1.0 (+https://usememos.com)"
- maxHTMLMetaBytes = 512 * 1024
+ initialHTMLMetaBytes = 512 * 1024
+ maxHTMLMetaBytes = 2 * 1024 * 1024
maxOEmbedBytes = 128 * 1024
maxCacheEntries = 1000
maxConcurrentFetches = 8
@@ -252,26 +254,53 @@ func (f *HTMLMetaFetcher) fetch(ctx context.Context, urlStr string) (*HTMLMeta,
return nil, errors.New("not a HTML page")
}
- document, err := html.Parse(io.LimitReader(response.Body, maxHTMLMetaBytes))
+ htmlBody, err := io.ReadAll(io.LimitReader(response.Body, initialHTMLMetaBytes))
if err != nil {
- return nil, errors.Wrap(err, "failed to parse HTML")
+ return nil, errors.Wrap(err, "failed to read HTML")
}
pageURL, err := finalResponseURL(response, urlStr)
if err != nil {
return nil, err
}
- sources, oEmbedEndpoint := extractDocumentMetadata(document)
- baseURL := resolveDocumentBase(pageURL, sources.baseHref)
- var oEmbed metadataSource
- if endpoint := resolveHTTPURL(baseURL, oEmbedEndpoint); endpoint != "" {
- // oEmbed is optional enrichment, so its failure must not discard the
- // metadata already extracted from the HTML document.
- oEmbed, _ = f.fetchOEmbed(ctx, endpoint)
+ var fetchedOEmbedEndpoint string
+ var fetchedOEmbed metadataSource
+ extract := func() (*HTMLMeta, error) {
+ document, err := html.Parse(bytes.NewReader(htmlBody))
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse HTML")
+ }
+ sources, oEmbedEndpoint := extractDocumentMetadata(document)
+ baseURL := resolveDocumentBase(pageURL, sources.baseHref)
+ var oEmbed metadataSource
+ if endpoint := resolveHTTPURL(baseURL, oEmbedEndpoint); endpoint != "" {
+ if endpoint != fetchedOEmbedEndpoint {
+ // oEmbed is optional enrichment, so its failure must not discard
+ // metadata already extracted from the HTML document.
+ fetchedOEmbed, _ = f.fetchOEmbed(ctx, endpoint)
+ fetchedOEmbedEndpoint = endpoint
+ }
+ oEmbed = fetchedOEmbed
+ }
+ return mergeMetadata(baseURL, oEmbed, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, sources.semantic), nil
}
- meta := mergeMetadata(baseURL, oEmbed, sources.openGraph, sources.twitter, sources.jsonLD, sources.standard, sources.semantic)
- return meta, nil
+ meta, err := extract()
+ if err != nil || (meta.Title != "" && meta.Image != "") || len(htmlBody) < initialHTMLMetaBytes {
+ return meta, err
+ }
+
+ // Metadata can follow a large inline script. Read further only when the
+ // initial document lacks a title or image, while keeping a hard byte cap.
+ rest, err := io.ReadAll(io.LimitReader(response.Body, maxHTMLMetaBytes-int64(len(htmlBody))))
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to read HTML")
+ }
+ if len(rest) == 0 {
+ return meta, nil
+ }
+ htmlBody = append(htmlBody, rest...)
+ return extract()
}
func (f *HTMLMetaFetcher) fetchOEmbed(ctx context.Context, endpoint string) (metadataSource, error) {
diff --git a/internal/linkmeta/html_meta_test.go b/internal/linkmeta/html_meta_test.go
index c1aadb3dd..6b4df7904 100644
--- a/internal/linkmeta/html_meta_test.go
+++ b/internal/linkmeta/html_meta_test.go
@@ -291,7 +291,7 @@ func TestHTMLMetaFetcherValidatesResponse(t *testing.T) {
}
func TestHTMLMetaFetcherBoundsHTMLResponse(t *testing.T) {
- body := `<title>Bounded title</title>` + strings.Repeat("x", maxHTMLMetaBytes) + `<meta property="og:title" content="Outside limit">`
+ body := `<title>Bounded title</title><script>` + strings.Repeat("x", maxHTMLMetaBytes) + `</script><meta property="og:image" content="/outside.png">`
reader := &countingReadCloser{Reader: strings.NewReader(body)}
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
result := response(req, http.StatusOK, "text/html", "")
@@ -302,9 +302,48 @@ func TestHTMLMetaFetcherBoundsHTMLResponse(t *testing.T) {
meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")
require.NoError(t, err)
require.Equal(t, "Bounded title", meta.Title)
+ require.Empty(t, meta.Image)
require.LessOrEqual(t, reader.bytesRead, int64(maxHTMLMetaBytes))
}
+func TestHTMLMetaFetcherStopsAfterInitialLimitWhenMetadataIsComplete(t *testing.T) {
+ body := `<meta property="og:title" content="Early title"><meta property="og:image" content="/early.png"><script>` + strings.Repeat("x", maxHTMLMetaBytes)
+ reader := &countingReadCloser{Reader: strings.NewReader(body)}
+ fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ result := response(req, http.StatusOK, "text/html", "")
+ result.Body = reader
+ return result, nil
+ }))
+
+ meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/page")
+ require.NoError(t, err)
+ require.Equal(t, "Early title", meta.Title)
+ require.Equal(t, "http://93.184.216.34/early.png", meta.Image)
+ require.LessOrEqual(t, reader.bytesRead, int64(initialHTMLMetaBytes))
+}
+
+func TestHTMLMetaFetcherFindsOEmbedPastInitialLimit(t *testing.T) {
+ body := `<html><head><script>` + strings.Repeat("x", initialHTMLMetaBytes) + `</script><link rel="alternate" type="application/json+oembed" href="/oembed"></head></html>`
+ var requests atomic.Int32
+ fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ requests.Add(1)
+ switch req.URL.Path {
+ case "/post":
+ return htmlResponse(req, body), nil
+ case "/oembed":
+ return response(req, http.StatusOK, "application/json", `{"title":"Late title","thumbnail_url":"/late.png"}`), nil
+ default:
+ return nil, errors.New("unexpected request path: " + req.URL.Path)
+ }
+ }))
+
+ meta, err := fetcher.Get(context.Background(), "http://93.184.216.34/post")
+ require.NoError(t, err)
+ require.Equal(t, "Late title", meta.Title)
+ require.Equal(t, "http://93.184.216.34/late.png", meta.Image)
+ require.EqualValues(t, 2, requests.Load())
+}
+
func TestHTMLMetaFetcherContextCancellation(t *testing.T) {
requestStarted := make(chan struct{})
fetcher := newTestFetcher(roundTripFunc(func(req *http.Request) (*http.Response, error) {