Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

12.9 KB
Raw Blame
export type MobileSharedPayload = {
  contentMimeType?: string | null;
  contentType?: string | null;
  contentUri?: string | null;
  mimeType?: string;
  originalName?: string | null;
  shareType?: string;
  value?: string;
};

export type MobileSharedImage = {
  mimeType: string;
  name: string;
  uri: string;
};

export type MobileWebClipDraft = {
  contentMarkdown: string;
  sourceUrl: string;
  tagsText: string;
  title: string;
};

export type MobileRenderedWebPage = {
  contentHtml: string;
  finalUrl?: string;
  title?: string;
};

export type MobileWebClipLocale = "zh-CN" | "en-US" | "ja" | "pl" | "ru";

const webClipCopy: Record<MobileWebClipLocale, {
  capturedAtLabel: string;
  fallbackTitle: string;
  imageAlt: string;
  separator: string;
  sourceLabel: string;
  unavailableBody: string;
}> = {
  "zh-CN": {
    sourceLabel: "来源",
    capturedAtLabel: "剪藏时间",
    unavailableBody: "正文暂时无法抓取,来源链接已保留,可稍后重试。",
    fallbackTitle: "网页剪藏",
    imageAlt: "图片",
    separator: ":",
  },
  "en-US": {
    sourceLabel: "Source",
    capturedAtLabel: "Captured at",
    unavailableBody: "Could not extract the page text. The source link is kept so you can retry later.",
    fallbackTitle: "Web clip",
    imageAlt: "Image",
    separator: ": ",
  },
  ja: {
    sourceLabel: "出典",
    capturedAtLabel: "取り込み日時",
    unavailableBody: "本文を取り込めませんでした。出典リンクは残してあるので、あとからやり直せます。",
    fallbackTitle: "ウェブクリップ",
    imageAlt: "画像",
    separator: ": ",
  },
  pl: {
    sourceLabel: "Źródło",
    capturedAtLabel: "Zapisano",
    unavailableBody: "Nie udało się wyodrębnić treści strony. Link do źródła został zachowany, więc możesz spróbować ponownie później.",
    fallbackTitle: "Wycinek strony",
    imageAlt: "Obraz",
    separator: ": ",
  },
  ru: {
    sourceLabel: "Источник",
    capturedAtLabel: "Сохранено",
    unavailableBody: "Не удалось извлечь текст страницы. Ссылка на источник сохранена, позже можно повторить.",
    fallbackTitle: "Вырезка страницы",
    imageAlt: "Изображение",
    separator: ": ",
  },
};

const URL_PATTERN = /https?:\/\/[^\s<>"'))]+/i;
const WECHAT_ARTICLE_HOSTS = new Set(["mp.weixin.qq.com"]);
const BLOCK_TAGS = new Set([
  "article",
  "aside",
  "blockquote",
  "div",
  "figure",
  "figcaption",
  "footer",
  "header",
  "main",
  "p",
  "section",
  "table",
  "tbody",
  "td",
  "th",
  "thead",
  "tr",
]);

const HTML_ENTITIES: Record<string, string> = {
  amp: "&",
  apos: "'",
  gt: ">",
  hellip: "…",
  ldquo: "“",
  lsquo: "‘",
  lt: "<",
  middot: "·",
  nbsp: " ",
  ndash: "–",
  quot: "\"",
  rdquo: "”",
  rsquo: "’",
};

export const getSharedWebUrl = (payloads: MobileSharedPayload[]) => {
  for (const payload of payloads) {
    const value = payload.value?.trim() ?? "";
    const match = value.match(URL_PATTERN);
    if (!match) {
      continue;
    }
    try {
      const url = new URL(match[0]);
      if (url.protocol === "http:" || url.protocol === "https:") {
        return url.toString();
      }
    } catch {
      // Ignore malformed text and continue looking through other payloads.
    }
  }
  return null;
};

export const getSharedImages = (payloads: MobileSharedPayload[]): MobileSharedImage[] =>
  payloads.flatMap((payload, index) => {
    const mimeType = payload.contentMimeType?.trim() || payload.mimeType?.trim() || "";
    const isImage = payload.contentType === "image" || payload.shareType === "image" || mimeType.startsWith("image/");
    const uri = payload.contentUri?.trim() || payload.value?.trim() || "";
    if (!isImage || !uri) {
      return [];
    }

    return [{
      mimeType: mimeType || "image/jpeg",
      name: payload.originalName?.trim() || `shared-image-${index + 1}`,
      uri,
    }];
  });

export const isWeChatArticleUrl = (value: string) => {
  try {
    const url = new URL(value);
    return url.protocol === "https:" && WECHAT_ARTICLE_HOSTS.has(url.hostname.toLowerCase()) && url.pathname.startsWith("/s");
  } catch {
    return false;
  }
};

export const buildMobileWebClipDraft = async (
  sourceUrl: string,
  options: {
    fetcher?: typeof fetch;
    capturedAt?: Date;
    locale?: MobileWebClipLocale;
  } = {}
): Promise<MobileWebClipDraft> => {
  const fetcher = options.fetcher ?? fetch;
  const capturedAt = options.capturedAt ?? new Date();
  const locale = options.locale ?? "zh-CN";
  let html = "";
  let finalUrl = sourceUrl;

  try {
    const response = await fetcher(sourceUrl, {
      headers: {
        Accept: "text/html,application/xhtml+xml",
      },
    });
    if (!response.ok) {
      throw new Error(`HTTP ${response.status}`);
    }
    finalUrl = response.url || sourceUrl;
    html = await response.text();
  } catch {
    return buildFallbackDraft(sourceUrl, capturedAt, locale);
  }

  const title = extractPageTitle(html) || hostnameTitle(sourceUrl, locale);
  const articleHtml = isWeChatArticleUrl(sourceUrl)
    ? extractElementInnerHtmlById(html, "js_content")
    : extractFirstElementInnerHtml(html, ["article", "main"]);
  return buildMobileWebClipDraftFromRenderedPage(
    sourceUrl,
    {
      contentHtml: articleHtml,
      finalUrl,
      title,
    },
    { capturedAt, locale },
  );
};

export const buildMobileWebClipDraftFromRenderedPage = (
  sourceUrl: string,
  page: MobileRenderedWebPage,
  options: {
    capturedAt?: Date;
    locale?: MobileWebClipLocale;
  } = {},
): MobileWebClipDraft => {
  const capturedAt = options.capturedAt ?? new Date();
  const locale = options.locale ?? "zh-CN";
  const copy = webClipCopy[locale];
  const title = normalizeInlineText(page.title ?? "") || hostnameTitle(sourceUrl, locale);
  const body = htmlToMarkdown(page.contentHtml, page.finalUrl || sourceUrl, copy.imageAlt);

  if (!body) {
    return { ...buildFallbackDraft(sourceUrl, capturedAt, locale), title };
  }

  return {
    title,
    sourceUrl,
    tagsText: isWeChatArticleUrl(sourceUrl) ? "web-clip, wechat" : "web-clip",
    contentMarkdown: [
      clipMetadataLine(copy.sourceLabel, `[${escapeMarkdownText(sourceUrl)}](${sourceUrl})`, copy.separator),
      clipMetadataLine(copy.capturedAtLabel, capturedAt.toISOString(), copy.separator),
      "---",
      body,
    ].join("\n\n"),
  };
};

export const extractPageTitle = (html: string) => {
  for (const property of ["og:title", "twitter:title"]) {
    const meta = findMetaContent(html, property);
    if (meta) {
      return normalizeInlineText(meta);
    }
  }

  const activityTitle = extractElementInnerHtmlById(html, "activity-name");
  if (activityTitle) {
    return normalizeInlineText(stripTags(activityTitle));
  }

  const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title\s*>/i);
  return titleMatch ? normalizeInlineText(stripTags(titleMatch[1])) : "";
};

export const htmlToMarkdown = (html: string, baseUrl?: string, imageAlt = "图片") => {
  if (!html.trim()) {
    return "";
  }

  let value = html
    .replace(/<!--[\s\S]*?-->/g, "")
    .replace(/<(script|style|noscript|template|iframe|svg|canvas)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "");

  value = value.replace(/<img\b[^>]*>/gi, (tag) => {
    const src = readAttribute(tag, "data-src") || readAttribute(tag, "data-original") || readAttribute(tag, "src");
    if (!src || src.startsWith("data:")) {
      return "";
    }
    const resolvedSrc = resolveWebUrl(src, baseUrl);
    if (!resolvedSrc) {
      return "";
    }
    const alt = normalizeInlineText(readAttribute(tag, "alt") || imageAlt);
    return `\n\n![${escapeMarkdownText(alt)}](${resolvedSrc})\n\n`;
  });

  value = value.replace(/<a\b[^>]*>([\s\S]*?)<\/a\s*>/gi, (full, inner: string) => {
    const href = readAttribute(full, "href");
    const label = normalizeInlineText(stripTags(inner));
    if (!href || /^javascript:/i.test(href)) {
      return label;
    }
    const resolvedHref = resolveWebUrl(href, baseUrl);
    return resolvedHref
      ? `[${escapeMarkdownText(label || resolvedHref)}](${resolvedHref})`
      : label;
  });

  value = value
    .replace(/<h([1-6])\b[^>]*>/gi, (_tag, level: string) => `\n\n${"#".repeat(Number(level))} `)
    .replace(/<\/h[1-6]\s*>/gi, "\n\n")
    .replace(/<br\s*\/?>/gi, "\n")
    .replace(/<hr\b[^>]*>/gi, "\n\n---\n\n")
    .replace(/<li\b[^>]*>/gi, "\n- ")
    .replace(/<\/li\s*>/gi, "\n")
    .replace(/<(strong|b)\b[^>]*>/gi, "**")
    .replace(/<\/(strong|b)\s*>/gi, "**")
    .replace(/<(em|i)\b[^>]*>/gi, "*")
    .replace(/<\/(em|i)\s*>/gi, "*")
    .replace(/<code\b[^>]*>/gi, "`")
    .replace(/<\/code\s*>/gi, "`")
    .replace(/<\/?blockquote\b[^>]*>/gi, "\n\n> ")
    .replace(/<\/?([a-z][a-z0-9-]*)\b[^>]*>/gi, (tag, name: string) =>
      BLOCK_TAGS.has(name.toLowerCase()) ? "\n\n" : tag.startsWith("</") ? "" : ""
    );

  return decodeHtmlEntities(value)
    .replace(/\r/g, "")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n[ \t]+/g, "\n")
    .replace(/[ \t]{2,}/g, " ")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
};

const clipMetadataLine = (label: string, value: string, separator: string) => `${label}${separator}${value}`;

const buildFallbackDraft = (
  sourceUrl: string,
  capturedAt: Date,
  locale: MobileWebClipLocale,
): MobileWebClipDraft => {
  const copy = webClipCopy[locale];
  return {
    title: hostnameTitle(sourceUrl, locale),
    sourceUrl,
    tagsText: isWeChatArticleUrl(sourceUrl) ? "web-clip, wechat" : "web-clip",
    contentMarkdown: [
      clipMetadataLine(copy.sourceLabel, `[${escapeMarkdownText(sourceUrl)}](${sourceUrl})`, copy.separator),
      clipMetadataLine(copy.capturedAtLabel, capturedAt.toISOString(), copy.separator),
      "",
      copy.unavailableBody,
    ].join("\n\n"),
  };
};

const hostnameTitle = (value: string, locale: MobileWebClipLocale = "zh-CN") => {
  try {
    return new URL(value).hostname;
  } catch {
    return webClipCopy[locale].fallbackTitle;
  }
};

const findMetaContent = (html: string, property: string) => {
  const tags = html.match(/<meta\b[^>]*>/gi) ?? [];
  for (const tag of tags) {
    const key = readAttribute(tag, "property") || readAttribute(tag, "name");
    if (key.toLowerCase() === property.toLowerCase()) {
      return decodeHtmlEntities(readAttribute(tag, "content"));
    }
  }
  return "";
};

const extractFirstElementInnerHtml = (html: string, tags: string[]) => {
  for (const tag of tags) {
    const match = html.match(new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}\\s*>`, "i"));
    if (match?.[1]) {
      return match[1];
    }
  }
  return "";
};

const extractElementInnerHtmlById = (html: string, id: string) => {
  const openingTagPattern = /<([a-z][a-z0-9-]*)\b[^>]*>/gi;
  let openingMatch: RegExpExecArray | null;
  while ((openingMatch = openingTagPattern.exec(html))) {
    if (readAttribute(openingMatch[0], "id") !== id) {
      continue;
    }
    const tagName = openingMatch[1].toLowerCase();
    const contentStart = openingTagPattern.lastIndex;
    const boundaryPattern = new RegExp(`<\\/?${tagName}\\b[^>]*>`, "gi");
    boundaryPattern.lastIndex = contentStart;
    let depth = 1;
    let boundaryMatch: RegExpExecArray | null;
    while ((boundaryMatch = boundaryPattern.exec(html))) {
      if (boundaryMatch[0].startsWith("</")) {
        depth -= 1;
        if (depth === 0) {
          return html.slice(contentStart, boundaryMatch.index);
        }
      } else if (!boundaryMatch[0].endsWith("/>")) {
        depth += 1;
      }
    }
    return html.slice(contentStart);
  }
  return "";
};

const readAttribute = (tag: string, name: string) => {
  const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const match = tag.match(new RegExp(`(?:^|\\s)${escapedName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i"));
  return decodeHtmlEntities(match?.[1] ?? match?.[2] ?? match?.[3] ?? "");
};

const stripTags = (value: string) => value.replace(/<[^>]*>/g, " ");
const normalizeInlineText = (value: string) => decodeHtmlEntities(value).replace(/\s+/g, " ").trim();
const escapeMarkdownText = (value: string) => value.replace(/([\\[\]])/g, "\\$1");
const resolveWebUrl = (value: string, baseUrl?: string) => {
  try {
    const url = baseUrl ? new URL(value, baseUrl) : new URL(value);
    if (url.protocol !== "http:" && url.protocol !== "https:") {
      return "";
    }
    return /^https?:\/\//i.test(value) ? value : url.toString();
  } catch {
    return "";
  }
};

const decodeHtmlEntities = (value: string) =>
  value.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]+);?/gi, (entity, key: string) => {
    if (key.startsWith("#x") || key.startsWith("#X")) {
      const codePoint = Number.parseInt(key.slice(2), 16);
      return Number.isFinite(codePoint) ? String.fromCodePoint(codePoint) : entity;
    }
    if (key.startsWith("#")) {
      const codePoint = Number.parseInt(key.slice(1), 10);
      return Number.isFinite(codePoint) ? String.fromCodePoint(codePoint) : entity;
    }
    return HTML_ENTITIES[key.toLowerCase()] ?? entity;
  });