Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

18.0 KB
Raw Blame
import Foundation
import SwiftUI
import WebKit

struct WebClipDraft: Equatable, Sendable {
    var title: String
    var contentMarkdown: String
    var tagsText: String

    var createSeed: CreateMemoSeed {
        CreateMemoSeed(title: title, contentMarkdown: contentMarkdown, tagsText: tagsText)
    }
}

struct RenderedWebPage: Equatable, Sendable {
    var title: String
    var contentHTML: String
    var finalURL: String
}

struct WebClipLabels: Equatable, Sendable {
    var sourceLabel: String
    var capturedAtLabel: String
    var unavailableBody: String
    var fallbackTitle: String
    var imageAlt: String
    var usesFullwidthColon: Bool

    static let chinese = WebClipLabels(
        sourceLabel: "来源",
        capturedAtLabel: "剪藏时间",
        unavailableBody: "正文暂时无法抓取,来源链接已保留,可稍后重试。",
        fallbackTitle: "网页剪藏",
        imageAlt: "图片",
        usesFullwidthColon: true
    )

    func labeledLine(_ label: String, value: String) -> String {
        usesFullwidthColon ? "\(label):\(value)" : "\(label): \(value)"
    }
}

enum WebClipper {
    static func sharedWebURL(from payloads: [ShareHandoffStore.SharePayload]) -> URL? {
        for payload in payloads {
            for candidate in [payload.url, payload.text].compactMap({ $0 }) {
                guard let range = candidate.range(of: #"https?://[^\s<>\"'))]+"#, options: .regularExpression),
                      let url = URL(string: String(candidate[range])),
                      url.scheme == "http" || url.scheme == "https"
                else { continue }
                return url
            }
        }
        return nil
    }

    static func isWeChatArticle(_ url: URL) -> Bool {
        url.scheme?.lowercased() == "https"
            && url.host?.lowercased() == "mp.weixin.qq.com"
            && url.path.hasPrefix("/s")
    }

    static func build(
        _ sourceURL: URL,
        capturedAt: Date = Date(),
        labels: WebClipLabels = .chinese,
        session: URLSession = .shared
    ) async -> WebClipDraft {
        var request = URLRequest(url: sourceURL)
        request.setValue("text/html,application/xhtml+xml", forHTTPHeaderField: "Accept")
        do {
            let (data, response) = try await session.data(for: request)
            guard let http = response as? HTTPURLResponse,
                  (200 ..< 300).contains(http.statusCode),
                  let html = String(data: data, encoding: .utf8)
                    ?? String(data: data, encoding: .isoLatin1)
            else { return fallback(sourceURL, capturedAt: capturedAt, labels: labels) }
            let finalURL = response.url ?? sourceURL
            let title = extractTitle(html) ?? hostnameTitle(sourceURL, fallbackTitle: labels.fallbackTitle)
            let bodyHTML = firstInnerHTML(html, tags: ["article", "main"])
            return buildRendered(
                sourceURL,
                page: RenderedWebPage(title: title, contentHTML: bodyHTML, finalURL: finalURL.absoluteString),
                capturedAt: capturedAt,
                labels: labels
            )
        } catch {
            return fallback(sourceURL, capturedAt: capturedAt, labels: labels)
        }
    }

    static func buildRendered(
        _ sourceURL: URL,
        page: RenderedWebPage,
        capturedAt: Date = Date(),
        labels: WebClipLabels = .chinese
    ) -> WebClipDraft {
        let title = normalizedText(page.title).nilIfEmpty ?? hostnameTitle(sourceURL, fallbackTitle: labels.fallbackTitle)
        let markdown = htmlToMarkdown(
            page.contentHTML,
            baseURL: URL(string: page.finalURL) ?? sourceURL,
            imageAlt: labels.imageAlt
        )
        guard !markdown.isEmpty else {
            var draft = fallback(sourceURL, capturedAt: capturedAt, labels: labels)
            draft.title = title
            return draft
        }
        let sourceLink = "[\(escapeMarkdown(sourceURL.absoluteString))](\(sourceURL.absoluteString))"
        return WebClipDraft(
            title: title,
            contentMarkdown: [
                labels.labeledLine(labels.sourceLabel, value: sourceLink),
                labels.labeledLine(labels.capturedAtLabel, value: ISO8601DateFormatter.edgeEver.string(from: capturedAt)),
                "---",
                markdown,
            ].joined(separator: "\n\n"),
            tagsText: isWeChatArticle(sourceURL) ? "web-clip, wechat" : "web-clip"
        )
    }

    static func fallback(
        _ sourceURL: URL,
        capturedAt: Date = Date(),
        labels: WebClipLabels = .chinese
    ) -> WebClipDraft {
        let sourceLink = "[\(escapeMarkdown(sourceURL.absoluteString))](\(sourceURL.absoluteString))"
        return WebClipDraft(
            title: hostnameTitle(sourceURL, fallbackTitle: labels.fallbackTitle),
            contentMarkdown: [
                labels.labeledLine(labels.sourceLabel, value: sourceLink),
                labels.labeledLine(labels.capturedAtLabel, value: ISO8601DateFormatter.edgeEver.string(from: capturedAt)),
                labels.unavailableBody,
            ].joined(separator: "\n\n"),
            tagsText: isWeChatArticle(sourceURL) ? "web-clip, wechat" : "web-clip"
        )
    }

    static func htmlToMarkdown(_ html: String, baseURL: URL, imageAlt: String = "图片") -> String {
        guard !html.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return "" }
        var value = replace(html, pattern: #"<!--[\s\S]*?-->"#) { _ in "" }
        value = replace(
            value,
            pattern: #"<(script|style|noscript|template|iframe|svg|canvas)\b[^>]*>[\s\S]*?</\1\s*>"#
        ) { _ in "" }
        value = replace(value, pattern: #"<img\b[^>]*>"#) { match in
            let source = attribute(match, "data-src").nilIfEmpty
                ?? attribute(match, "data-original").nilIfEmpty
                ?? attribute(match, "src").nilIfEmpty
            guard let source, !source.hasPrefix("data:"), let resolved = resolve(source, against: baseURL) else { return "" }
            let alt = normalizedText(attribute(match, "alt")).nilIfEmpty ?? imageAlt
            return "\n\n![\(escapeMarkdown(alt))](\(resolved))\n\n"
        }
        value = replace(value, pattern: #"<a\b[^>]*>([\s\S]*?)</a\s*>"#) { match in
            let href = attribute(match, "href")
            let label = normalizedText(stripTags(capture(match, pattern: #"<a\b[^>]*>([\s\S]*?)</a\s*>"#) ?? ""))
            guard !href.isEmpty, !href.lowercased().hasPrefix("javascript:"),
                  let resolved = resolve(href, against: baseURL)
            else { return label }
            return "[\(escapeMarkdown(label.isEmpty ? resolved : label))](\(resolved))"
        }
        value = replace(value, pattern: #"<h([1-6])\b[^>]*>"#) { match in
            let level = Int(capture(match, pattern: #"<h([1-6])\b"#) ?? "1") ?? 1
            return "\n\n\(String(repeating: "#", count: level)) "
        }
        let substitutions: [(String, String)] = [
            (#"</h[1-6]\s*>"#, "\n\n"), (#"<br\s*/?>"#, "\n"),
            (#"<hr\b[^>]*>"#, "\n\n---\n\n"), (#"<li\b[^>]*>"#, "\n- "),
            (#"</li\s*>"#, "\n"), (#"<(strong|b)\b[^>]*>"#, "**"),
            (#"</(strong|b)\s*>"#, "**"), (#"<(em|i)\b[^>]*>"#, "*"),
            (#"</(em|i)\s*>"#, "*"), (#"<code\b[^>]*>"#, "`"),
            (#"</code\s*>"#, "`"), (#"</?blockquote\b[^>]*>"#, "\n\n> "),
            (#"</?(article|aside|div|figure|figcaption|footer|header|main|p|section|table|tbody|td|th|thead|tr)\b[^>]*>"#, "\n\n"),
            (#"</?[a-z][a-z0-9-]*\b[^>]*>"#, ""),
        ]
        for (pattern, replacement) in substitutions {
            value = replace(value, pattern: pattern) { _ in replacement }
        }
        return decodeEntities(value)
            .replacingOccurrences(of: "\r", with: "")
            .replacingOccurrences(of: #"[ \t]+\n"#, with: "\n", options: .regularExpression)
            .replacingOccurrences(of: #"\n[ \t]+"#, with: "\n", options: .regularExpression)
            .replacingOccurrences(of: #"[ \t]{2,}"#, with: " ", options: .regularExpression)
            .replacingOccurrences(of: #"\n{3,}"#, with: "\n\n", options: .regularExpression)
            .trimmingCharacters(in: .whitespacesAndNewlines)
    }

    private static func extractTitle(_ html: String) -> String? {
        let metas = matches(html, pattern: #"<meta\b[^>]*>"#)
        for key in ["og:title", "twitter:title"] {
            if let tag = metas.first(where: {
                (attribute($0, "property").nilIfEmpty ?? attribute($0, "name")).lowercased() == key
            }), let title = normalizedText(attribute(tag, "content")).nilIfEmpty { return title }
        }
        guard let title = capture(html, pattern: #"<title\b[^>]*>([\s\S]*?)</title\s*>"#) else { return nil }
        return normalizedText(stripTags(title)).nilIfEmpty
    }

    private static func firstInnerHTML(_ html: String, tags: [String]) -> String {
        for tag in tags {
            if let value = capture(html, pattern: "<\(tag)\\b[^>]*>([\\s\\S]*?)</\(tag)\\s*>") { return value }
        }
        return ""
    }

    private static func hostnameTitle(_ url: URL, fallbackTitle: String) -> String { url.host ?? fallbackTitle }
    private static func normalizedText(_ value: String) -> String {
        decodeEntities(value).replacingOccurrences(of: #"\s+"#, with: " ", options: .regularExpression)
            .trimmingCharacters(in: .whitespacesAndNewlines)
    }
    private static func stripTags(_ value: String) -> String {
        value.replacingOccurrences(of: #"<[^>]*>"#, with: " ", options: .regularExpression)
    }
    private static func escapeMarkdown(_ value: String) -> String {
        value.replacingOccurrences(of: #"([\[\]])"#, with: #"\\$1"#, options: .regularExpression)
    }
    private static func resolve(_ value: String, against baseURL: URL) -> String? {
        guard let url = URL(string: value, relativeTo: baseURL)?.absoluteURL,
              url.scheme == "http" || url.scheme == "https" else { return nil }
        return url.absoluteString
    }
    private static func attribute(_ tag: String, _ name: String) -> String {
        let pattern = "(?:^|\\s)\(NSRegularExpression.escapedPattern(for: name))\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s>]+))"
        guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive),
              let match = regex.firstMatch(in: tag, range: NSRange(tag.startIndex..., in: tag)) else { return "" }
        for index in 1 ..< match.numberOfRanges where match.range(at: index).location != NSNotFound {
            if let range = Range(match.range(at: index), in: tag) { return decodeEntities(String(tag[range])) }
        }
        return ""
    }
    private static func capture(_ value: String, pattern: String) -> String? {
        guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive),
              let match = regex.firstMatch(in: value, range: NSRange(value.startIndex..., in: value)),
              match.numberOfRanges > 1, let range = Range(match.range(at: 1), in: value) else { return nil }
        return String(value[range])
    }
    private static func matches(_ value: String, pattern: String) -> [String] {
        guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) else { return [] }
        return regex.matches(in: value, range: NSRange(value.startIndex..., in: value)).compactMap {
            Range($0.range, in: value).map { String(value[$0]) }
        }
    }
    private static func replace(_ value: String, pattern: String, transform: (String) -> String) -> String {
        guard let regex = try? NSRegularExpression(pattern: pattern, options: .caseInsensitive) else { return value }
        var output = value
        for match in regex.matches(in: value, range: NSRange(value.startIndex..., in: value)).reversed() {
            guard let originalRange = Range(match.range, in: value), let outputRange = Range(match.range, in: output) else { continue }
            output.replaceSubrange(outputRange, with: transform(String(value[originalRange])))
        }
        return output
    }
    private static func decodeEntities(_ value: String) -> String {
        let named = ["amp": "&", "apos": "'", "gt": ">", "hellip": "…", "ldquo": "“", "lsquo": "‘", "lt": "<", "middot": "·", "nbsp": " ", "ndash": "–", "quot": "\"", "rdquo": "”", "rsquo": "’"]
        return replace(value, pattern: #"&(#x?[0-9a-f]+|[a-z][a-z0-9]+);?"#) { entity in
            let key = String(entity.dropFirst().dropLast(entity.hasSuffix(";") ? 1 : 0))
            if key.lowercased().hasPrefix("#x"), let scalar = UInt32(key.dropFirst(2), radix: 16).flatMap(UnicodeScalar.init) { return String(scalar) }
            if key.hasPrefix("#"), let scalar = UInt32(key.dropFirst()).flatMap(UnicodeScalar.init) { return String(scalar) }
            return named[key.lowercased()] ?? entity
        }
    }
}

/// Hidden rendered-page capture used for WeChat pages that server-side fetch cannot reliably read.
struct WebClipCaptureView: UIViewRepresentable {
    let url: URL
    let onCaptured: (RenderedWebPage) -> Void
    let onFailed: (String) -> Void

    func makeCoordinator() -> Coordinator { Coordinator(onCaptured: onCaptured, onFailed: onFailed) }

    func makeUIView(context: Context) -> WKWebView {
        let configuration = WKWebViewConfiguration()
        configuration.websiteDataStore = .default()
        configuration.userContentController.add(context.coordinator, name: "webClip")
        let view = WKWebView(frame: .zero, configuration: configuration)
        view.navigationDelegate = context.coordinator
        view.isOpaque = false
        view.backgroundColor = .clear
        return view
    }

    func updateUIView(_ view: WKWebView, context: Context) {
        context.coordinator.onCaptured = onCaptured
        context.coordinator.onFailed = onFailed
        guard context.coordinator.loadedURL != url else { return }
        context.coordinator.loadedURL = url
        view.load(URLRequest(url: url))
        context.coordinator.startTimeout()
    }

    static func dismantleUIView(_ view: WKWebView, coordinator: Coordinator) {
        coordinator.timeout?.cancel()
        view.stopLoading()
        view.configuration.userContentController.removeScriptMessageHandler(forName: "webClip")
    }

    final class Coordinator: NSObject, WKNavigationDelegate, WKScriptMessageHandler {
        var onCaptured: (RenderedWebPage) -> Void
        var onFailed: (String) -> Void
        var loadedURL: URL?
        var timeout: DispatchWorkItem?
        private var completed = false

        init(onCaptured: @escaping (RenderedWebPage) -> Void, onFailed: @escaping (String) -> Void) {
            self.onCaptured = onCaptured
            self.onFailed = onFailed
        }

        func startTimeout() {
            completed = false
            timeout?.cancel()
            let work = DispatchWorkItem { [weak self] in self?.finishFailure("微信文章加载超时。") }
            timeout = work
            DispatchQueue.main.asyncAfter(deadline: .now() + 20, execute: work)
        }

        func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) {
            webView.evaluateJavaScript(Self.captureScript, completionHandler: nil)
        }

        func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: Error) {
            finishFailure(error.localizedDescription)
        }

        func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, withError error: Error) {
            finishFailure(error.localizedDescription)
        }

        func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) {
            guard !completed, let body = message.body as? [String: Any], let status = body["status"] as? String else { return }
            if status == "captured", let html = body["contentHtml"] as? String, !html.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
                completed = true
                timeout?.cancel()
                onCaptured(RenderedWebPage(
                    title: body["title"] as? String ?? "",
                    contentHTML: html,
                    finalURL: body["finalUrl"] as? String ?? loadedURL?.absoluteString ?? ""
                ))
            } else {
                finishFailure(body["message"] as? String ?? "没有找到可剪藏的正文。")
            }
        }

        private func finishFailure(_ message: String) {
            guard !completed else { return }
            completed = true
            timeout?.cancel()
            onFailed(message)
        }

        private static let captureScript = #"""
        (function(){
          var attempts=0, completed=false;
          function send(value){ if(completed)return; completed=true; window.webkit.messageHandlers.webClip.postMessage(value); }
          function text(selector){ var el=document.querySelector(selector); return el&&el.textContent?el.textContent.trim():""; }
          function capture(){
            var content=document.getElementById("js_content")||document.querySelector("article")||document.querySelector("main");
            if(content && content.textContent && content.textContent.trim()){
              var meta=document.querySelector('meta[property="og:title"],meta[name="twitter:title"]');
              send({status:"captured",title:(meta&&meta.getAttribute("content"))||text("#activity-name")||text(".rich_media_title")||document.title||"",contentHtml:content.innerHTML,finalUrl:location.href}); return;
            }
            var error=text(".weui-msg__title")||text(".weui-msg__desc");
            if(error){send({status:"failed",message:error});return;}
            attempts++; if(attempts>=50){send({status:"failed",message:"页面加载完成,但没有找到可剪藏的正文。"});return;}
            setTimeout(capture,250);
          }
          capture();
        })();
        """#
    }
}

private extension String {
    var nilIfEmpty: String? {
        let value = trimmingCharacters(in: .whitespacesAndNewlines)
        return value.isEmpty ? nil : value
    }
}