Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

11.1 KB
Raw Blame
package linkmeta

import (
	"bytes"
	"encoding/json"
	"mime"
	"net/url"
	"slices"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/pkg/errors"
	"golang.org/x/net/html"
)

const (
	maxTitleRunes        = 200
	maxDescriptionRunes  = 300
	minSemanticTextRunes = 40
	maxImageURLBytes     = 2048
)

type metadataSource struct {
	title       string
	description string
	image       string
}

type documentMetadata struct {
	openGraph metadataSource
	twitter   metadataSource
	jsonLD    metadataSource
	standard  metadataSource
	semantic  metadataSource
	baseHref  string
}

func extractDocumentMetadata(document *html.Node) (documentMetadata, string) {
	var sources documentMetadata
	var oEmbedEndpoint string
	var jsonLDScripts []string

	walkNodes(document, func(node *html.Node) {
		if node.Type != html.ElementNode {
			return
		}
		switch strings.ToLower(node.Data) {
		case "title":
			// Foreign-content titles (inline SVG/MathML accessibility labels)
			// are not the document title.
			if node.Namespace == "" {
				setIfEmpty(&sources.standard.title, nodeText(node))
			}
		case "base":
			setIfEmpty(&sources.baseHref, firstAttribute(node, "href"))
		case "meta":
			name := strings.ToLower(strings.TrimSpace(firstAttribute(node, "property", "name")))
			content := firstAttribute(node, "content")
			switch name {
			case "og:title":
				setIfEmpty(&sources.openGraph.title, content)
			case "og:description":
				setIfEmpty(&sources.openGraph.description, content)
			case "og:image", "og:image:url", "og:image:secure_url":
				setImageIfEmpty(&sources.openGraph.image, content)
			case "twitter:title":
				setIfEmpty(&sources.twitter.title, content)
			case "twitter:description":
				setIfEmpty(&sources.twitter.description, content)
			case "twitter:image", "twitter:image:src":
				setImageIfEmpty(&sources.twitter.image, content)
			case "description":
				setIfEmpty(&sources.standard.description, content)
			case "title", "name", "headline":
				setIfEmpty(&sources.standard.title, content)
			case "image":
				setImageIfEmpty(&sources.standard.image, content)
			default:
			}
		case "link":
			rels := strings.Fields(strings.ToLower(firstAttribute(node, "rel")))
			if containsString(rels, "image_src") {
				setImageIfEmpty(&sources.standard.image, firstAttribute(node, "href"))
			}
			if oEmbedEndpoint == "" && containsString(rels, "alternate") {
				mediaType := firstAttribute(node, "type")
				if isJSONOEmbedMediaType(mediaType) {
					oEmbedEndpoint = firstAttribute(node, "href")
				}
			}
		case "script":
			if isJSONLDMediaType(firstAttribute(node, "type")) {
				jsonLDScripts = append(jsonLDScripts, rawNodeText(node))
			}
		default:
		}
	})

	sources.jsonLD = extractJSONLDMetadata(jsonLDScripts)
	sources.semantic = extractSemanticMetadata(document)
	return sources, oEmbedEndpoint
}

func mergeMetadata(pageURL *url.URL, sources ...metadataSource) *HTMLMeta {
	meta := &HTMLMeta{}
	for _, source := range sources {
		setIfEmpty(&meta.Title, source.title)
		setIfEmpty(&meta.Description, source.description)
		if meta.Image == "" {
			meta.Image = resolveHTTPURL(pageURL, source.image)
		}
	}
	meta.Title = truncateText(normalizeWhitespace(meta.Title), maxTitleRunes)
	meta.Description = truncateText(normalizeWhitespace(meta.Description), maxDescriptionRunes)
	// A truncated URL would be broken, so oversized image URLs are dropped
	// rather than shortened.
	if len(meta.Image) > maxImageURLBytes {
		meta.Image = ""
	}
	return meta
}

func extractOEmbedMetadata(data []byte, baseURL *url.URL) (metadataSource, error) {
	var payload struct {
		Type         string `json:"type"`
		Title        string `json:"title"`
		Description  string `json:"description"`
		ThumbnailURL string `json:"thumbnail_url"`
		URL          string `json:"url"`
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	if err := decoder.Decode(&payload); err != nil {
		return metadataSource{}, errors.Wrap(err, "failed to decode oEmbed response")
	}
	image := resolveHTTPURL(baseURL, payload.ThumbnailURL)
	if image == "" && strings.EqualFold(payload.Type, "photo") {
		image = resolveHTTPURL(baseURL, payload.URL)
	}
	return metadataSource{
		title:       payload.Title,
		description: payload.Description,
		image:       image,
	}, nil
}

func extractJSONLDMetadata(scripts []string) metadataSource {
	var best metadataSource
	bestScore := -1
	for _, script := range scripts {
		var value any
		decoder := json.NewDecoder(strings.NewReader(script))
		if err := decoder.Decode(&value); err != nil {
			continue
		}
		walkJSONLD(value, func(object map[string]any) {
			candidate := metadataSource{
				title:       firstJSONString(object["headline"], object["name"]),
				description: firstJSONString(object["description"]),
				image:       jsonLDImage(object["image"]),
			}
			if candidate.image == "" {
				candidate.image = jsonLDImage(object["thumbnailUrl"])
			}
			if candidate == (metadataSource{}) {
				return
			}
			score := jsonLDTypeScore(object["@type"])
			if candidate.title != "" {
				score++
			}
			if candidate.description != "" {
				score++
			}
			if candidate.image != "" {
				score++
			}
			if score > bestScore {
				best = candidate
				bestScore = score
			}
		})
	}
	return best
}

func walkJSONLD(value any, visit func(map[string]any)) {
	switch typed := value.(type) {
	case []any:
		for _, item := range typed {
			walkJSONLD(item, visit)
		}
	case map[string]any:
		visit(typed)
		for _, nested := range typed {
			walkJSONLD(nested, visit)
		}
	default:
	}
}

func jsonLDTypeScore(value any) int {
	types := jsonLDStrings(value)
	for _, value := range types {
		switch strings.ToLower(value) {
		case "article", "newsarticle", "blogposting", "socialmediaposting", "product", "videoobject", "movie", "book", "event", "recipe":
			return 20
		case "creativework", "webpage", "profilepage", "aboutpage", "collectionpage":
			return 10
		case "breadcrumblist", "itemlist":
			return -10
		}
	}
	return 0
}

func jsonLDImage(value any) string {
	switch typed := value.(type) {
	case string:
		if isPotentialHTTPURL(typed) {
			return typed
		}
	case []any:
		for _, item := range typed {
			if image := jsonLDImage(item); image != "" {
				return image
			}
		}
	case map[string]any:
		// "@id" is deliberately excluded: it is a node reference (often the
		// page's own URL plus a fragment), not an image location.
		for _, candidate := range []any{typed["url"], typed["contentUrl"]} {
			if image := jsonLDImage(candidate); image != "" {
				return image
			}
		}
	default:
	}
	return ""
}

func firstJSONString(values ...any) string {
	for _, value := range values {
		for _, candidate := range jsonLDStrings(value) {
			if strings.TrimSpace(candidate) != "" {
				return candidate
			}
		}
	}
	return ""
}

func jsonLDStrings(value any) []string {
	switch typed := value.(type) {
	case string:
		return []string{typed}
	case []any:
		var result []string
		for _, item := range typed {
			result = append(result, jsonLDStrings(item)...)
		}
		return result
	}
	return nil
}

func extractSemanticMetadata(document *html.Node) metadataSource {
	root := firstElement(document, "main", "article")
	if root == nil {
		root = firstElement(document, "body")
	}
	if root == nil {
		return metadataSource{}
	}

	var source metadataSource
	walkNodes(root, func(node *html.Node) {
		if node.Type != html.ElementNode {
			return
		}
		switch strings.ToLower(node.Data) {
		case "h1":
			setIfEmpty(&source.title, nodeText(node))
		case "p":
			text := normalizeWhitespace(nodeText(node))
			if source.description == "" && utf8.RuneCountInString(text) >= minSemanticTextRunes {
				source.description = text
			}
		case "img":
			setImageIfEmpty(&source.image, firstAttribute(node, "src"))
		default:
		}
	})
	return source
}

func resolveHTTPURL(baseURL *url.URL, candidate string) string {
	candidate = strings.TrimSpace(candidate)
	if candidate == "" {
		return ""
	}
	parsed, err := url.Parse(candidate)
	if err != nil {
		return ""
	}
	if baseURL != nil {
		parsed = baseURL.ResolveReference(parsed)
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return ""
	}
	if parsed.Hostname() == "" {
		return ""
	}
	return parsed.String()
}

func walkNodes(node *html.Node, visit func(*html.Node)) {
	if node == nil {
		return
	}
	visit(node)
	for child := node.FirstChild; child != nil; child = child.NextSibling {
		walkNodes(child, visit)
	}
}

// firstElement prefers earlier names over document order: all matches for
// names[0] outrank any match for names[1].
func firstElement(node *html.Node, names ...string) *html.Node {
	for _, name := range names {
		var result *html.Node
		walkNodes(node, func(candidate *html.Node) {
			if result != nil || candidate.Type != html.ElementNode {
				return
			}
			if strings.EqualFold(candidate.Data, name) {
				result = candidate
			}
		})
		if result != nil {
			return result
		}
	}
	return nil
}

func firstAttribute(node *html.Node, names ...string) string {
	for _, name := range names {
		for _, attribute := range node.Attr {
			if strings.EqualFold(attribute.Key, name) {
				return attribute.Val
			}
		}
	}
	return ""
}

func rawNodeText(node *html.Node) string {
	var builder strings.Builder
	walkNodes(node, func(candidate *html.Node) {
		if candidate.Type == html.TextNode {
			builder.WriteString(candidate.Data)
		}
	})
	return builder.String()
}

func nodeText(node *html.Node) string {
	return normalizeWhitespace(rawNodeText(node))
}

func normalizeWhitespace(value string) string {
	return strings.Join(strings.FieldsFunc(value, unicode.IsSpace), " ")
}

func truncateText(value string, maximum int) string {
	runes := []rune(value)
	if len(runes) <= maximum {
		return value
	}
	return strings.TrimSpace(string(runes[:maximum-1])) + "…"
}

func setIfEmpty(target *string, value string) {
	if *target == "" && strings.TrimSpace(value) != "" {
		*target = value
	}
}

func setImageIfEmpty(target *string, value string) {
	if *target == "" && isPotentialHTTPURL(value) {
		*target = value
	}
}

func isPotentialHTTPURL(value string) bool {
	value = strings.TrimSpace(value)
	if value == "" {
		return false
	}
	parsed, err := url.Parse(value)
	if err != nil {
		return false
	}
	if parsed.Scheme == "" {
		return !strings.HasPrefix(value, "//") || parsed.Hostname() != ""
	}
	return (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Hostname() != ""
}

func containsString(values []string, expected string) bool {
	return slices.Contains(values, expected)
}

func isJSONOEmbedMediaType(value string) bool {
	mediaType, _, err := mime.ParseMediaType(value)
	return err == nil && strings.EqualFold(mediaType, "application/json+oembed")
}

func isJSONLDMediaType(value string) bool {
	mediaType, _, err := mime.ParseMediaType(value)
	return err == nil && strings.EqualFold(mediaType, "application/ld+json")
}

// resolveDocumentBase applies the document's <base href>, falling back to the
// final response URL when the base is missing or not an absolute http(s) URL.
func resolveDocumentBase(pageURL *url.URL, baseHref string) *url.URL {
	baseHref = strings.TrimSpace(baseHref)
	if baseHref == "" {
		return pageURL
	}
	parsed, err := url.Parse(baseHref)
	if err != nil {
		return pageURL
	}
	resolved := pageURL.ResolveReference(parsed)
	if (resolved.Scheme != "http" && resolved.Scheme != "https") || resolved.Hostname() == "" {
		return pageURL
	}
	return resolved
}