AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He
新增 web-to-md.cjs 并完善文档
hugohe3c6dbbcbc05e2
diff --git a/AGENTS.md b/AGENTS.mdindex 334bde86..0511c7d0 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -228,7 +228,7 @@ Optimizer_CRAP (可选)python3 tools/pdf_to_md.py <PDF文件># 网页转 Markdown(抓取网页内容并保存图片)-python3 tools/web_to_md.py <URL>+python3 tools/web_to_md.py <URL> 或 node tools/web_to_md.cjs <URL># 初始化项目python3 tools/project_manager.py init <名称> --format ppt169@@ -295,7 +295,8 @@ project/| 场景 | 推荐工具 | 命令 || ------------------------------- | --------------- | ------------------------------------- |-| **新闻/文章类网页** | `web_to_md.py` | `python3 tools/web_to_md.py <URL>` |+| **微信公众号/高防站点** | `web_to_md.cjs` | `node tools/web_to_md.cjs <URL>` (绕过 TLS 拦截,推荐) |+| **普通文章/新闻网页** | `web_to_md.py` | `python3 tools/web_to_md.py <URL>` || **图文内容**(游记、攻略等) | `web_to_md.py` | 同上(自动下载图片到 `_files/`) || **政府/机构网站** | `web_to_md.py` | 同上(支持中文站点元数据提取) || **批量处理多个 URL** | `web_to_md.py` | `python3 tools/web_to_md.py -f urls.txt` |diff --git a/tools/README.md b/tools/README.mdindex fda2e4bd..7cccb7cc 100644--- a/tools/README.md+++ b/tools/README.md@@ -53,10 +53,12 @@ pip install PyMuPDF----### 0.5. web_to_md.py — 网页转 Markdown 工具+### 0.5. web_to_md.py / web_to_md.cjs — 网页转 Markdown 工具将网页内容抓取并转换为 Markdown 格式,自动下载图片到本地。+**注意**: 提供 Python 和 Node.js 两个版本。对于微信公众号等有 TLS 指纹拦截的站点,**强烈推荐使用 Node.js 版本** (`web_to_md.cjs`)。+**功能**:- 抓取网页内容并转换为 Markdown@@ -65,7 +67,7 @@ pip install PyMuPDF- 智能识别主要内容区域(支持中文新闻/政府网站)- 支持批量处理多个 URL-**用法**:+**用法 (Python)**:```bash# 转换单个网页@@ -81,6 +83,19 @@ python3 tools/web_to_md.py -f urls.txtpython3 tools/web_to_md.py https://example.com -o output.md```+**用法 (Node.js) — 推荐用于微信公众号**:++```bash+# 转换单个网页+node tools/web_to_md.cjs https://mp.weixin.qq.com/s/xxxx++# 转换多个网页+node tools/web_to_md.cjs https://url1.com https://url2.com++# 从文件批量读取 URL+node tools/web_to_md.cjs -f urls.txt+```+**输出结构**:```@@ -92,24 +107,29 @@ projects/└── ...```-**何时使用 web_to_md.py**:+**何时使用 web_to_md.py / .cjs**:| 场景 | 推荐工具 | 原因 ||------|----------|------|-| **新闻/文章类网页** | `web_to_md.py` | 自动提取正文、下载图片 |-| **图文内容**(游记、攻略等) | `web_to_md.py` | 保留图片资源 |-| **政府/机构网站** | `web_to_md.py` | 支持中文站点元数据提取 |+| **微信公众号/高防站点** | `web_to_md.cjs` | Node.js 对 TLS 指纹拦截不仅更友好,且能大大降低被 403 的概率 |+| **普通新闻/文章类网页** | 任选其一 | 均可自动提取正文、下载图片 |+| **图文内容**(游记、攻略等) | 任选其一 | 保留图片资源 |+| **政府/机构网站** | `web_to_md.py` | Python 版对某些特定中文编码处理可能微优 || **需要登录的页面** | 手动处理 | 工具不支持认证 || **动态渲染页面(SPA)** | 手动处理 | 需要 headless browser |-> **策略**: 静态网页用 `web_to_md.py`,动态渲染或需登录的页面需手动处理。+> **策略**: 遇到 403 错误或微信文章时,请直接切换使用 `web_to_md.cjs`。**依赖**:+Python:```bashpip install requests beautifulsoup4```+Node.js:+(脚本使用原生模块,无需额外 npm install,但需 Node.js 环境)+---### 1. project_utils.py — 项目工具公共模块diff --git a/tools/web_to_md.cjs b/tools/web_to_md.cjsnew file mode 100644index 00000000..e59ddaff--- /dev/null+++ b/tools/web_to_md.cjs@@ -0,0 +1,847 @@+#!/usr/bin/env node+/**+ * web_to_md.cjs - Web Page to Markdown Converter (Enhanced)+ *+ * Usage:+ * node tools/web_to_md.cjs <url> # Single URL+ * node tools/web_to_md.cjs <url1> <url2> ... # Multiple URLs+ * node tools/web_to_md.cjs -f urls.txt # Read URLs from file+ * node tools/web_to_md.cjs <url> -o output.md # Specify output filename+ */++const fs = require("fs").promises;+const path = require("path");+const https = require("https");+const http = require("http");++// ============ Config ============+const CONFIG = {+ outputDir: "./projects",+ timeout: 30000,+ userAgent:+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",+};++// ============ HTTP Fetch with encoding detection ============+async function fetchUrl(url) {+ return new Promise((resolve, reject) => {+ const client = url.startsWith("https") ? https : http;+ const options = {+ headers: {+ "User-Agent": CONFIG.userAgent,+ Accept:+ "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",+ },+ timeout: CONFIG.timeout,+ };++ const req = client.get(url, options, (res) => {+ // Handle redirects+ if (+ res.statusCode >= 300 &&+ res.statusCode < 400 &&+ res.headers.location+ ) {+ const redirectUrl = new URL(res.headers.location, url).href;+ return fetchUrl(redirectUrl).then(resolve).catch(reject);+ }++ if (res.statusCode !== 200) {+ reject(new Error("HTTP " + res.statusCode));+ return;+ }++ const chunks = [];+ res.on("data", (chunk) => chunks.push(chunk));+ res.on("end", () => {+ const buffer = Buffer.concat(chunks);++ // Detect encoding from Content-Type header or meta tag+ let encoding = "utf-8";+ const contentType = res.headers["content-type"] || "";+ const charsetMatch = contentType.match(/charset=([^;]+)/i);+ if (charsetMatch) {+ encoding = charsetMatch[1].trim().toLowerCase();+ }++ // Try UTF-8 first, then check for charset in HTML+ let html = buffer.toString("utf-8");++ // Check for meta charset in HTML+ const metaCharsetMatch = html.match(+ /<meta[^>]*charset=["']?([^"'\s>]+)/i+ );+ if (metaCharsetMatch) {+ const declaredEncoding = metaCharsetMatch[1].toLowerCase();+ if (+ declaredEncoding === "gbk" ||+ declaredEncoding === "gb2312" ||+ declaredEncoding === "gb18030"+ ) {+ // For Chinese GBK encoding, we'd need iconv-lite+ // For now, UTF-8 usually works for modern sites+ }+ }++ resolve(html);+ });+ });++ req.on("error", reject);+ req.on("timeout", () => {+ req.destroy();+ reject(new Error("Request timeout"));+ });+ });+}++// ============ Enhanced HTML Parser ============+function parseHTML(html) {+ // Extract title+ const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);+ let title = titleMatch ? decodeHTMLEntities(titleMatch[1].trim()) : "";+ // Clean title - remove site name suffix+ title = title.replace(/[-_|].*?(政府|门户|网站|委员会).*$/g, "").trim();++ // Extract meta tags+ const metas = {};+ const metaRegex = /<meta[^>]+>/gi;+ let metaMatch;+ while ((metaMatch = metaRegex.exec(html)) !== null) {+ const tag = metaMatch[0];+ const nameMatch = tag.match(/(?:name|property)=["']([^"']+)["']/i);+ const contentMatch = tag.match(/content=["']([^"']+)["']/i);+ if (nameMatch && contentMatch) {+ metas[nameMatch[1].toLowerCase()] = decodeHTMLEntities(contentMatch[1]);+ }+ }++ // Enhanced content extraction - try multiple strategies+ let content = extractMainContent(html);++ return { title, metas, content };+}++// ============ Enhanced Main Content Extraction ============+function extractMainContent(html) {+ // Remove unwanted elements first+ let cleanHtml = html;++ // Remove script, style, nav, header, footer, sidebar, comments+ cleanHtml = cleanHtml.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");+ cleanHtml = cleanHtml.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");+ cleanHtml = cleanHtml.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, "");+ cleanHtml = cleanHtml.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, "");+ cleanHtml = cleanHtml.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, "");+ cleanHtml = cleanHtml.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, "");+ cleanHtml = cleanHtml.replace(/<!--[\s\S]*?-->/g, "");++ // Common content container patterns for Chinese government websites+ const contentPatterns = [+ // Hunan province gov sites+ /<div[^>]*class=["'][^"']*tys-main-zt-show[^"']*["'][^>]?>([\s\S]*?)<\/div>\s*<\/div>\s*<\/div>/i,+ /<div[^>]*class=["'][^"']*tys-main-zt-show[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*tys-main[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ // Exact class matches for government sites+ /<div[^>]*class=["'][^"']*TRS_Editor[^"']*["'][^>]*>([\s\S]*?)<\/div>\s*(?:<\/div>)?/i,+ /<div[^>]*class=["'][^"']*TRS_UEDITOR[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*ucontent[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*article-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*news-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*detail-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*content-text[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*pages_content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*zwgk_content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*content_detail[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*text_content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*main-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*main_content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*view-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*class=["'][^"']*info-content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*id=["']?Zoom["']?[^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*id=["']?content["']?[^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*id=["']?article["']?[^>]*>([\s\S]*?)<\/div>/i,+ // ID-based patterns+ /<div[^>]*id=["'][^"']*content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ /<div[^>]*id=["'][^"']*article[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ // Standard semantic elements+ /<article[^>]*>([\s\S]*?)<\/article>/i,+ /<main[^>]*>([\s\S]*?)<\/main>/i,+ // Generic content divs (less specific)+ /<div[^>]*class=["'][^"']*content[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,+ ];++ // Try each pattern and find the one with most text content+ let bestContent = "";+ let bestTextLength = 0;++ for (const pattern of contentPatterns) {+ // Use a greedy matching approach for nested divs+ const matches = findAllMatches(cleanHtml, pattern);+ for (const match of matches) {+ const textContent = stripTags(match).trim();+ // Prefer content with Chinese characters (for gov sites)+ const chineseChars = (textContent.match(/[\u4e00-\u9fa5]/g) || []).length;+ const score = textContent.length + chineseChars * 2;+ if (score > bestTextLength && textContent.length > 200) {+ bestContent = match;+ bestTextLength = score;+ }+ }+ }++ // If still not found, try to find content between common markers+ if (!bestContent || bestTextLength < 500) {+ // Look for content area by finding dense paragraph areas+ const bodyMatch = cleanHtml.match(/<body[^>]*>([\s\S]*?)<\/body>/i);+ if (bodyMatch) {+ const bodyContent = bodyMatch[1];+ // Find the area with most <p> tags+ const areas = bodyContent.split(/<div[^>]*>/i);+ for (const area of areas) {+ const pCount = (area.match(/<p[^>]*>/gi) || []).length;+ const textLen = stripTags(area).length;+ if (pCount >= 2 && textLen > bestTextLength) {+ bestContent = area;+ bestTextLength = textLen;+ }+ }+ }+ }++ // Fallback to body+ if (!bestContent) {+ const bodyMatch = cleanHtml.match(/<body[^>]*>([\s\S]*?)<\/body>/i);+ bestContent = bodyMatch ? bodyMatch[1] : cleanHtml;+ }++ return bestContent;+}++// Helper function to find all regex matches+function findAllMatches(html, pattern) {+ const results = [];+ const globalPattern = new RegExp(pattern.source, "gi");+ let match;+ while ((match = globalPattern.exec(html)) !== null) {+ if (match[1]) {+ results.push(match[1]);+ }+ }+ return results;+}++// Helper function to strip HTML tags for text length calculation+function stripTags(html) {+ return html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ");+}++// ============ HTML Entity Decoder ============+function decodeHTMLEntities(str) {+ const entities = {+ "&": "&",+ "<": "<",+ ">": ">",+ """: '"',+ "'": "'",+ " ": " ",+ "—": "\u2014",+ "–": "\u2013",+ "©": "\u00A9",+ "®": "\u00AE",+ "™": "\u2122",+ "…": "\u2026",+ "“": "\u201C",+ "”": "\u201D",+ "‘": "\u2018",+ "’": "\u2019",+ " ": " ",+ " ": " ",+ "·": "\u00B7",+ "•": "\u2022",+ "×": "\u00D7",+ "÷": "\u00F7",+ "±": "\u00B1",+ "«": "\u00AB",+ "»": "\u00BB",+ };+ // Handle numeric entities+ str = str.replace(/&#(\d+);/g, function (match, dec) {+ return String.fromCharCode(dec);+ });+ str = str.replace(/&#x([0-9a-fA-F]+);/g, function (match, hex) {+ return String.fromCharCode(parseInt(hex, 16));+ });+ // Handle named entities+ return str.replace(/&[a-zA-Z]+;/g, function (match) {+ return entities[match] || match;+ });+}++// ============ Enhanced HTML to Markdown Converter ============+function htmlToMarkdown(html) {+ let md = html;++ // First pass: remove unwanted elements+ md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");+ md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");+ md = md.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, "");+ md = md.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, "");+ md = md.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, "");+ md = md.replace(/<!--[\s\S]*?-->/g, "");++ // Convert headings+ md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, "\n# $1\n");+ md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, "\n## $1\n");+ md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, "\n### $1\n");+ md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, "\n#### $1\n");+ md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, "\n##### $1\n");+ md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, "\n###### $1\n");++ // Convert paragraphs+ md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, "\n$1\n");+ md = md.replace(/<div[^>]*>([\s\S]*?)<\/div>/gi, "\n$1\n");+ md = md.replace(/<br\s*\/?>/gi, "\n");+ md = md.replace(/<hr\s*\/?>/gi, "\n---\n");++ // Convert text formatting+ md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/\1>/gi, "**$2**");+ md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/\1>/gi, "*$2*");+ md = md.replace(/<u[^>]*>([\s\S]*?)<\/u>/gi, "$1");+ md = md.replace(/<del[^>]*>([\s\S]*?)<\/del>/gi, "~~$1~~");+ md = md.replace(/<s[^>]*>([\s\S]*?)<\/s>/gi, "~~$1~~");+ md = md.replace(/<span[^>]*>([\s\S]*?)<\/span>/gi, "$1");++ // Convert links - handle nested content properly+ md = md.replace(+ /<a[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi,+ function (match, url, text) {+ const cleanText = text.replace(/<[^>]+>/g, "").trim();+ if (!cleanText || url.startsWith("javascript:")) {+ return cleanText;+ }+ return "[" + cleanText + "](" + url + ")";+ }+ );++ // Convert images+ md = md.replace(+ /<img[^>]*src=["']([^"']+)["'][^>]*alt=["']([^"']*)["'][^>]*\/?>/gi,+ ""+ );+ md = md.replace(+ /<img[^>]*alt=["']([^"']*)["'][^>]*src=["']([^"']+)["'][^>]*\/?>/gi,+ ""+ );+ md = md.replace(/<img[^>]*src=["']([^"']+)["'][^>]*\/?>/gi, "");++ // Convert lists+ md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, function (match, content) {+ const cleanContent = content.replace(/<[^>]+>/g, "").trim();+ return "- " + cleanContent + "\n";+ });+ md = md.replace(/<\/?[uo]l[^>]*>/gi, "\n");++ // Convert code+ md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, "`$1`");+ md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, "\n```\n$1\n```\n");++ // Convert blockquotes+ md = md.replace(+ /<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi,+ function (match, content) {+ const lines = content.replace(/<[^>]+>/g, "").split("\n");+ return (+ "\n" ++ lines+ .map(function (line) {+ return "> " + line.trim();+ })+ .filter(function (l) {+ return l.trim() !== ">";+ })+ .join("\n") ++ "\n"+ );+ }+ );++ // Convert tables+ md = md.replace(+ /<table[^>]*>([\s\S]*?)<\/table>/gi,+ function (match, content) {+ let result = "\n";+ const rows = content.match(/<tr[^>]*>[\s\S]*?<\/tr>/gi) || [];+ rows.forEach(function (row, index) {+ const cells = row.match(/<t[hd][^>]*>([\s\S]*?)<\/t[hd]>/gi) || [];+ const cellContents = cells.map(function (cell) {+ const cellMatch = cell.match(/<t[hd][^>]*>([\s\S]*?)<\/t[hd]>/i);+ return cellMatch ? cellMatch[1].replace(/<[^>]+>/g, "").trim() : "";+ });+ if (cellContents.length > 0) {+ result += "| " + cellContents.join(" | ") + " |\n";+ if (index === 0) {+ result +=+ "| " ++ cellContents+ .map(function () {+ return "---";+ })+ .join(" | ") ++ " |\n";+ }+ }+ });+ return result;+ }+ );++ // Clean remaining HTML tags+ md = md.replace(/<[^>]+>/g, "");++ // Decode HTML entities+ md = decodeHTMLEntities(md);++ // Clean up whitespace+ md = md.replace(/\r\n/g, "\n");+ md = md.replace(/\n{3,}/g, "\n\n");+ md = md.replace(/[ \t]+$/gm, "");+ md = md.replace(/^[ \t]+/gm, "");++ // Remove lines that are just whitespace or single characters+ md = md+ .split("\n")+ .filter(function (line) {+ const trimmed = line.trim();+ return trimmed.length > 1 || trimmed === "" || /^[-#>*|]/.test(trimmed);+ })+ .join("\n");++ md = md.trim();++ return md;+}++// ============ Extract Metadata ============+function extractMetadata(parsed, url) {+ const title = parsed.title;+ const metas = parsed.metas;++ // Try to extract date from multiple sources+ let date =+ metas["article:published_time"] ||+ metas["og:published_time"] ||+ metas["pubdate"] ||+ metas["publishdate"] ||+ metas["date"] ||+ "";++ // Try to extract date from HTML content+ if (!date) {+ const content = parsed.content;+ // Common date patterns in Chinese gov sites+ const datePatterns = [+ /发布[时日]间[::]\s*(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)/,+ /日期[::]\s*(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)/,+ /(\d{4}[-\/年]\d{1,2}[-\/月]\d{1,2}[日]?)\s*(?:发布|来源)/,+ /时间[::]\s*(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})/,+ ];+ for (const pattern of datePatterns) {+ const match = content.match(pattern);+ if (match) {+ date = match[1].replace(/[年月]/g, "-").replace(/[日]/g, "");+ break;+ }+ }+ }++ // Try to extract date from URL+ if (!date) {+ const urlDateMatch = url.match(/(\d{4})(\d{2})[\/_](?:t\d+_)?/i);+ if (urlDateMatch) {+ date = urlDateMatch[1] + "-" + urlDateMatch[2];+ } else {+ const urlDateMatch2 = url.match(/(\d{4})[-\/](\d{2})[-\/](\d{2})/);+ if (urlDateMatch2) {+ date =+ urlDateMatch2[1] + "-" + urlDateMatch2[2] + "-" + urlDateMatch2[3];+ }+ }+ }++ const description =+ metas["description"] ||+ metas["og:description"] ||+ metas["twitter:description"] ||+ "";++ const author = metas["author"] || metas["article:author"] || "";++ const keywords = metas["keywords"] || "";++ // Extract source/publisher from content+ let source = "";+ const sourcePatterns = [+ /来源[::]\s*([^\s<]+)/,+ /发布(?:单位|机构)[::]\s*([^\s<]+)/,+ ];+ for (const pattern of sourcePatterns) {+ const match = parsed.content.match(pattern);+ if (match) {+ source = match[1];+ break;+ }+ }++ return {+ title: title || metas["og:title"] || "",+ date: date,+ description: description,+ author: author || source,+ keywords: keywords,+ sourceUrl: url,+ };+}++// ============ Sanitize Filename ============+function sanitizeFilename(name) {+ return name+ .replace(/[<>:"/\\|?*]/g, "")+ .replace(/\s+/g, "_")+ .replace(/_+/g, "_")+ .substring(0, 80);+}++// ============ Image Downloader & Processor ============+function downloadImage(url, savePath) {+ return new Promise((resolve, reject) => {+ const client = url.startsWith("https") ? https : http;+ const req = client.get(+ url,+ {+ headers: {+ "User-Agent": CONFIG.userAgent,+ Accept: "image/*,*/*",+ },+ timeout: CONFIG.timeout,+ },+ (res) => {+ if (res.statusCode !== 200) {+ res.resume();+ return reject(new Error("Status " + res.statusCode));+ }+ const fileStream = require("fs").createWriteStream(savePath);+ res.pipe(fileStream);+ fileStream.on("finish", () => {+ fileStream.close();+ resolve();+ });+ fileStream.on("error", (err) => {+ reject(err);+ });+ }+ );+ req.on("error", (err) => reject(err));+ req.on("timeout", () => {+ req.destroy();+ reject(new Error("Timeout"));+ });+ });+}++function buildImageFilename(urlStr, index) {+ try {+ const u = new URL(urlStr);+ const basename = path.basename(u.pathname);+ let ext = path.extname(basename);+ let name = path.basename(basename, ext);++ name = sanitizeFilename(name);+ if (!name) name = "image_" + index;+ // Basic extension check+ if (!ext || ext.length > 5 || !/^\.[a-z0-9]+$/i.test(ext)) ext = ".jpg";++ return name + ext;+ } catch (e) {+ return "image_" + index + ".jpg";+ }+}++async function processImages(html, pageUrl, imageDir, relPrefix) {+ const imgRegex = /<img[^>]+src=["']([^"']+)["'][^>]*>/gi;+ const matches = [...html.matchAll(imgRegex)];++ if (matches.length === 0) return html;++ console.log(" [Processing] Found " + matches.length + " images...");++ // Ensure image dir exists+ await fs.mkdir(imageDir, { recursive: true });++ const urlMap = new Map();+ const uniqueUrls = new Set();++ for (const m of matches) {+ if (m[1]) uniqueUrls.add(m[1]);+ }++ let idx = 0;+ for (const src of uniqueUrls) {+ if (src.startsWith("data:") || src.startsWith("#")) continue;++ try {+ // Resolve absolute URL+ const absUrl = new URL(src, pageUrl).href;+ // Generate filename+ const filename = buildImageFilename(absUrl, idx++);++ // Avoid overwriting existing files with same name but different content?+ // For now, simpler collision avoidance+ let saveName = filename;+ let counter = 1;++ // Note: This collision check is basic.+ while (true) {+ try {+ await fs.access(path.join(imageDir, saveName));+ const parsed = path.parse(filename);+ saveName = parsed.name + "_" + counter++ + parsed.ext;+ } catch {+ break; // file does not exist+ }+ }++ const savePath = path.join(imageDir, saveName);+ await downloadImage(absUrl, savePath);++ // Relative path for Markdown+ const relPath = relPrefix+ ? path.join(relPrefix, saveName)+ : saveName;++ // Force forward slashes for Markdown compatibility+ urlMap.set(src, relPath.replace(/\\/g, "/"));+ } catch (e) {+ console.log(" [WARN] Image fail " + src + ": " + e.message);+ }+ }++ // Replace src in HTML+ return html.replace(imgRegex, (match, src) => {+ if (urlMap.has(src)) {+ // Replace only the src part of the match+ // Simplest robust way is to replace the exact src string within the match+ // but we must be careful not to replace other attributes' values if identical.+ // A safer way is to reconstruct, but regex is tricky.+ // Current approach: replace the FIRST occurrence of src in the match?+ // No, src is what we extracted.++ // Let's replace `src="URL"` with `src="NEW_URL"`?+ // Match structure is roughly `<img ... src="URL" ...>`+ // We can look for `src=["']URL["']` inside `match`.++ // Create a regex for this specific src in this tag+ // Escape special regex chars in src+ const escapedSrc = src.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');+ const srcReplaceRegex = new RegExp(`(src=["'])(${escapedSrc})(["'])`, "i");+ return match.replace(srcReplaceRegex, `$1${urlMap.get(src)}$3`);+ }+ return match;+ });+}++// ============ Process Single URL ============+// ============ Process Single URL ============+async function processUrl(url, outputPath) {+ console.log("\n[Fetching] " + url);++ try {+ const html = await fetchUrl(url);+ console.log(" [OK] Fetched (" + (html.length / 1024).toFixed(1) + " KB)");++ const parsed = parseHTML(html);+ const metadata = extractMetadata(parsed, url);+ console.log(" [OK] Title: " + (metadata.title || "(not found)"));+ if (metadata.date) console.log(" [OK] Date: " + metadata.date);++ // Determine output path EARLY so we can determine image path+ if (!outputPath) {+ await fs.mkdir(CONFIG.outputDir, { recursive: true });+ const filename = sanitizeFilename(metadata.title || "untitled") + ".md";+ outputPath = path.join(CONFIG.outputDir, filename);+ }++ // Set up image directory+ const outputDir = path.dirname(outputPath);+ await fs.mkdir(outputDir, { recursive: true });++ const baseName = path.basename(outputPath, path.extname(outputPath));+ const imageDirName = baseName + "_files"; // Using string concatenation+ const imageDir = path.join(outputDir, imageDirName);+ const relImagePrefix = imageDirName;++ // Process Images+ let contentToConvert = parsed.content;+ try {+ contentToConvert = await processImages(parsed.content, url, imageDir, relImagePrefix);+ } catch (e) {+ console.warn(" [WARN] Image processing failed, continuing: " + e.message);+ }++ const markdown = htmlToMarkdown(contentToConvert);+ console.log(" [OK] Content: " + markdown.length + " chars");++ // Generate output+ let output = "";+ output += "<!-- \n";+ output += " Source: " + url + "\n";+ output += " Crawled: " + new Date().toISOString() + "\n";+ if (metadata.date) output += " Published: " + metadata.date + "\n";+ if (metadata.author) output += " Author: " + metadata.author + "\n";+ output += "-->\n\n";++ if (metadata.title) {+ output += "# " + metadata.title + "\n\n";+ }++ if (metadata.description) {+ output += "> " + metadata.description + "\n\n";+ }++ output += markdown;++ await fs.writeFile(outputPath, output, "utf-8");+ console.log(" [OK] Saved: " + outputPath);++ return {+ success: true,+ url: url,+ outputPath: outputPath,+ metadata: metadata,+ };+ } catch (error) {+ console.error(" [ERROR] " + error.message);+ return { success: false, url: url, error: error.message };+ }+}++// ============ Main Function ============+async function main() {+ const args = process.argv.slice(2);++ if (args.length === 0) {+ console.log("");+ console.log("=".repeat(60));+ console.log(" Web to Markdown - Enhanced Web Page Crawler & Converter");+ console.log("=".repeat(60));+ console.log("");+ console.log(" Usage:");+ console.log(" node tools/web_to_md.cjs <url>");+ console.log(" node tools/web_to_md.cjs <url1> <url2> ...");+ console.log(" node tools/web_to_md.cjs -f urls.txt");+ console.log(" node tools/web_to_md.cjs <url> -o output.md");+ console.log("");+ console.log(" Options:");+ console.log(" -f, --file <file> Read URLs from file (one per line)");+ console.log(+ " -o, --output <file> Specify output filename (single URL only)"+ );+ console.log(+ " -d, --dir <dir> Specify output directory (default: ./output)"+ );+ console.log(" -h, --help Show help");+ console.log("");+ console.log("=".repeat(60));+ process.exit(0);+ }++ let urls = [];+ let outputFile = null;++ // Parse arguments+ for (let i = 0; i < args.length; i++) {+ const arg = args[i];++ if (arg === "-h" || arg === "--help") {+ args.length = 0;+ await main();+ return;+ }++ if (arg === "-f" || arg === "--file") {+ const filePath = args[++i];+ if (!filePath) {+ console.error("Error: -f requires a file path");+ process.exit(1);+ }+ const content = await fs.readFile(filePath, "utf-8");+ const lines = content+ .split("\n")+ .map(function (l) {+ return l.trim();+ })+ .filter(function (l) {+ return l && !l.startsWith("#");+ });+ urls = urls.concat(lines);+ continue;+ }++ if (arg === "-o" || arg === "--output") {+ outputFile = args[++i];+ continue;+ }++ if (arg === "-d" || arg === "--dir") {+ CONFIG.outputDir = args[++i];+ continue;+ }++ if (arg.startsWith("http://") || arg.startsWith("https://")) {+ urls.push(arg);+ }+ }++ if (urls.length === 0) {+ console.error("Error: No valid URLs provided");+ process.exit(1);+ }++ console.log("\n[Start] Processing " + urls.length + " URL(s)...");++ const results = [];+ for (let i = 0; i < urls.length; i++) {+ const result = await processUrl(+ urls[i],+ urls.length === 1 ? outputFile : null+ );+ results.push(result);+ }++ // Summary+ const success = results.filter(function (r) {+ return r.success;+ }).length;+ const failed = results.filter(function (r) {+ return !r.success;+ }).length;++ console.log("\n" + "=".repeat(50));+ console.log(+ "[Done] Success: " + success + "/" + urls.length + ", Failed: " + failed+ );++ if (failed > 0) {+ console.log("\n[Failed URLs]:");+ results+ .filter(function (r) {+ return !r.success;+ })+ .forEach(function (r) {+ console.log(" - " + r.url + ": " + r.error);+ });+ }+}++main().catch(console.error);diff --git a/tools/web_to_md.py b/tools/web_to_md.pyindex 479ba912..b5150397 100644--- a/tools/web_to_md.py+++ b/tools/web_to_md.py@@ -11,6 +11,12 @@ Usage:Dependencies:pip install requests beautifulsoup4++Note:+ Some websites (e.g., WeChat mp.weixin.qq.com) block Python's 'requests' library+ based on TLS fingerprints (JA3). If you encounter 403 errors or connection issues+ with such sites, please use the Node.js version (tools/web_to_md.cjs) or+ modify this script to use 'curl_cffi'."""import argparse