Skip to content

欢迎来到电子书下载宝库,一个汇聚了各类电子书下载链接的地方。无论你是喜欢阅读经典文学、经管励志、终身学习、职场创业、技术手册还是其他类型的书籍,这里都能满足你的需求。 该库涵盖了帆书app(原樊登读书)、微信读书、京东读书、喜马拉雅等读书app的大部分电子书。

Python
13.8 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
批量测试脚本:根据书籍ID批量处理并生成md文件
测试范围:ID 1-1000
"""

import asyncio
import aiohttp
import time
from pathlib import Path
from collections import defaultdict
from typing import Dict, List, Set
import json
import sys

# 添加当前目录到路径
sys.path.insert(0, str(Path(__file__).parent))

from parse_book_detail_enhanced import parse_book_detail_enhanced

# 尝试导入配置文件,如果不存在则使用环境变量
import os
try:
    from config import BOOK_SITE_DOMAIN
except ImportError:
    # 如果配置文件不存在,尝试从环境变量读取(GitHub Actions会使用这种方式)
    BOOK_SITE_DOMAIN = os.getenv("BOOK_SITE_DOMAIN", "")
    if not BOOK_SITE_DOMAIN:
        print("⚠️  警告:未找到配置文件 config.py,也未设置环境变量 BOOK_SITE_DOMAIN")
        print("   本地运行:请复制 config.py.example 为 config.py 并配置 BOOK_SITE_DOMAIN")
        print("   GitHub Actions:请在 Repository Settings > Secrets 中添加 BOOK_SITE_DOMAIN")
        print("   或设置环境变量: export BOOK_SITE_DOMAIN='https://www.dushupai.com'")
        sys.exit(1)

# 配置
BASE_URL = f"{BOOK_SITE_DOMAIN}/book-content-{{}}.html"
# 输出目录:根据环境变量决定是测试目录还是正式目录
OUTPUT_DIR_ENV = os.getenv("OUTPUT_DIR", "md_test")
if OUTPUT_DIR_ENV == "md":
    OUTPUT_DIR = Path(__file__).parent.parent.parent / "md"  # 正式目录
else:
    OUTPUT_DIR = Path(__file__).parent.parent.parent / "md_test"  # 测试目录
PROCESSED_IDS_FILE = OUTPUT_DIR / "processed_ids.json"
STATS_FILE = OUTPUT_DIR / "stats.json"
MAX_BOOK_ID_FILE = OUTPUT_DIR / "max_book_id.json"  # 记录最大书籍ID

# 并发配置
MAX_CONCURRENT = 20  # 最大并发数
REQUEST_DELAY = 0.5  # 请求延迟(秒)


def sanitize_filename(filename: str) -> str:
    """
    清理文件名,移除不合法字符
    
    Args:
        filename: 原始文件名
    
    Returns:
        清理后的文件名
    """
    # 替换不合法字符
    invalid_chars = ['/', '\\', ':', '*', '?', '"', '<', '>', '|']
    for char in invalid_chars:
        filename = filename.replace(char, '_')
    
    # 移除首尾空格和点
    filename = filename.strip(' .')
    
    # 限制长度
    if len(filename) > 200:
        filename = filename[:200]
    
    return filename or "未命名"


def generate_md_file(tag_name: str, books: List[Dict], output_dir: Path) -> Path:
    """
    生成Markdown文件
    
    Args:
        tag_name: 标签名称
        books: 书籍列表
        output_dir: 输出目录
    
    Returns:
        生成的文件路径
    """
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # 清理文件名
    safe_filename = sanitize_filename(tag_name)
    file_path = output_dir / f"{safe_filename}.md"
    
    # 过滤有效书籍(必须有书名和下载链接)
    valid_books = []
    for book in books:
        title = book.get('title', '').strip()
        # 优先使用实际下载链接(诚通网盘链接),避免使用下载页面链接(包含敏感域名)
        download_url = book.get('download_url', '').strip()
        
        # 如果没有实际下载链接,跳过该书(不包含下载页面链接以保护隐私)
        if not download_url:
            continue
        
        if title and download_url:
            valid_books.append({
                'title': title,
                'author': book.get('author', '未知').strip() or '未知',
                'download_url': download_url
            })
    
    if not valid_books:
        return None
    
    # 生成Markdown内容
    lines = []
    lines.append("| 书名 | 作者 | epub/mobi/azw3 |")
    lines.append("| --- | --- | --- |")
    
    for book in valid_books:
        # 转义Markdown特殊字符
        title_escaped = book['title'].replace('|', '\\|')
        author_escaped = book['author'].replace('|', '\\|')
        # 修改下载链接:将 ?pwd= 改成 ?p=
        download_url = book['download_url'].replace('?pwd=', '?p=')
        
        # 隐私保护:如果下载链接包含敏感域名,移除该书籍(不生成到md文件)
        # 这样可以确保GitHub仓库中不包含敏感域名
        if download_url and BOOK_SITE_DOMAIN in download_url:
            # 跳过包含敏感域名的链接(通常是下载页面链接)
            # 只保留诚通网盘的实际下载链接
            continue
        
        download_link = f"[下载]({download_url})"
        
        lines.append(f"| {title_escaped} | {author_escaped} | {download_link} |")
    
    # 写入文件
    file_path.write_text('\n'.join(lines), encoding='utf-8')
    return file_path


async def fetch_book_async(session: aiohttp.ClientSession, book_id: int, semaphore: asyncio.Semaphore) -> Dict:
    """
    异步获取书籍信息
    
    Args:
        session: aiohttp会话
        book_id: 书籍ID
        semaphore: 信号量控制并发
    
    Returns:
        书籍信息字典,失败返回None
    """
    async with semaphore:
        url = BASE_URL.format(book_id)
        
        try:
            # 添加延迟避免请求过快
            await asyncio.sleep(REQUEST_DELAY)
            
            # 使用同步请求(因为parse_book_detail_enhanced是同步的)
            # 在实际生产环境中,可以改为异步版本
            loop = asyncio.get_event_loop()
            result = await loop.run_in_executor(None, parse_book_detail_enhanced, url)
            
            # 检查是否成功获取到书名(判断书籍是否存在)
            if result.get('title'):
                result['book_id'] = str(book_id)
                return result
            else:
                return None
                
        except Exception as e:
            print(f"❌ 处理书籍ID {book_id} 时出错: {e}")
            return None


async def batch_process_books(book_ids: List[int]) -> Dict[str, List[Dict]]:
    """
    批量处理书籍
    
    Args:
        book_ids: 书籍ID列表
    
    Returns:
        按标签分类的书籍字典 {tag: [books]}
    """
    # 按标签分类的书籍
    books_by_tag = defaultdict(list)
    
    # 创建信号量控制并发
    semaphore = asyncio.Semaphore(MAX_CONCURRENT)
    
    # 创建aiohttp会话
    async with aiohttp.ClientSession() as session:
        # 创建任务列表
        tasks = [fetch_book_async(session, book_id, semaphore) for book_id in book_ids]
        
        # 处理结果
        completed = 0
        total = len(tasks)
        
        for coro in asyncio.as_completed(tasks):
            book_data = await coro
            completed += 1
            
            if book_data:
                # 按标签分类
                tags = book_data.get('tags', [])
                if tags:
                    for tag in tags:
                        books_by_tag[tag].append(book_data)
                else:
                    # 如果没有标签,使用分类作为标签
                    category = book_data.get('category', '未分类')
                    if category:
                        books_by_tag[category].append(book_data)
                    else:
                        books_by_tag['未分类'].append(book_data)
                
                # 显示进度
                if completed % 10 == 0 or completed == total:
                    print(f"📊 进度: {completed}/{total} ({completed*100//total}%) - 已找到 {len(books_by_tag)} 个标签")
            else:
                if completed % 50 == 0 or completed == total:
                    print(f"📊 进度: {completed}/{total} ({completed*100//total}%)")
    
    return dict(books_by_tag)


def save_processed_ids(book_ids: Set[int]):
    """保存已处理的ID列表"""
    PROCESSED_IDS_FILE.parent.mkdir(parents=True, exist_ok=True)
    with open(PROCESSED_IDS_FILE, 'w', encoding='utf-8') as f:
        json.dump(sorted(book_ids), f, ensure_ascii=False, indent=2)


def load_processed_ids() -> Set[int]:
    """加载已处理的ID列表"""
    if PROCESSED_IDS_FILE.exists():
        with open(PROCESSED_IDS_FILE, 'r', encoding='utf-8') as f:
            return set(json.load(f))
    return set()


def save_stats(stats: Dict):
    """保存统计信息"""
    with open(STATS_FILE, 'w', encoding='utf-8') as f:
        json.dump(stats, f, ensure_ascii=False, indent=2)


def save_max_book_id(max_id: int):
    """保存最大书籍ID"""
    with open(MAX_BOOK_ID_FILE, 'w', encoding='utf-8') as f:
        json.dump({'max_book_id': max_id, 'last_updated': time.strftime('%Y-%m-%d %H:%M:%S')}, 
                 f, ensure_ascii=False, indent=2)


def load_max_book_id() -> int:
    """加载最大书籍ID"""
    if MAX_BOOK_ID_FILE.exists():
        with open(MAX_BOOK_ID_FILE, 'r', encoding='utf-8') as f:
            data = json.load(f)
            return data.get('max_book_id', 0)
    return 0


def generate_hot_categories_index(books_by_tag: Dict[str, List[Dict]], output_dir: Path) -> Path:
    """
    生成热门分类索引文件(按照README.md格式)
    
    Args:
        books_by_tag: 按标签分类的书籍字典
        output_dir: 输出目录
    
    Returns:
        生成的文件路径
    """
    output_dir.mkdir(parents=True, exist_ok=True)
    file_path = output_dir / "热门分类.md"
    
    # 按书籍数量排序
    sorted_tags = sorted(books_by_tag.items(), key=lambda x: len(x[1]), reverse=True)
    
    # 生成内容
    lines = []
    lines.append("# 📚 热门分类")
    lines.append("")
    lines.append("> **💡 使用 Ctrl+F 快速搜索关键词,或点击下方标签直接进入分类页面**")
    lines.append("")
    lines.append("## 🔥 热门分类")
    lines.append("")
    
    # 每行8个标签
    items_per_line = 8
    for i in range(0, len(sorted_tags), items_per_line):
        batch = sorted_tags[i:i + items_per_line]
        line_items = []
        
        for tag, books in batch:
            count = len(books)
            # 清理文件名(与generate_md_file中的逻辑一致)
            safe_filename = sanitize_filename(tag)
            link = f"- [{tag}({count})]({safe_filename}.md)"
            line_items.append(link)
        
        # 用 | 分隔,每行8个
        line = "  | ".join(line_items)
        lines.append(line)
    
    # 写入文件
    file_path.write_text('\n'.join(lines), encoding='utf-8')
    return file_path


async def main(start_id: int = 1, end_id: int = 1000):
    """
    主函数
    
    Args:
        start_id: 起始书籍ID(默认:1)
        end_id: 结束书籍ID(默认:1000)
    """
    print("=" * 80)
    print(f"🚀 开始批量处理书籍(ID: {start_id}-{end_id})")
    print("=" * 80)
    
    # 创建输出目录
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    print(f"📁 输出目录: {OUTPUT_DIR}")
    
    # 生成书籍ID列表
    book_ids = list(range(start_id, end_id + 1))
    print(f"📚 待处理书籍数量: {len(book_ids)}")
    
    # 加载已处理的ID(用于断点续传)
    processed_ids = load_processed_ids()
    if processed_ids:
        print(f"📋 已处理ID数量: {len(processed_ids)}")
        book_ids = [bid for bid in book_ids if bid not in processed_ids]
        print(f"📚 剩余待处理: {len(book_ids)}")
    
    if not book_ids:
        print("✅ 所有书籍已处理完成!")
        return
    
    # 开始处理
    start_time = time.time()
    books_by_tag = await batch_process_books(book_ids)
    elapsed_time = time.time() - start_time
    
    # 统计信息
    total_books = sum(len(books) for books in books_by_tag.values())
    total_tags = len(books_by_tag)
    
    print(f"\n📊 处理完成!")
    print(f"  - 总耗时: {elapsed_time:.2f} 秒")
    print(f"  - 成功处理: {total_books} 本书")
    print(f"  - 标签数量: {total_tags} 个")
    
    # 生成md文件
    print(f"\n📝 开始生成Markdown文件...")
    generated_files = []
    
    for tag, books in sorted(books_by_tag.items()):
        file_path = generate_md_file(tag, books, OUTPUT_DIR)
        if file_path:
            generated_files.append(str(file_path))
            # 如果文件数量很多,减少输出频率
            if len(generated_files) <= 50 or len(generated_files) % 50 == 0:
                print(f"  ✅ {tag}: {len(books)} 本书 -> {file_path.name}")
    
    print(f"\n✅ 共生成 {len(generated_files)} 个Markdown文件")
    
    # 保存统计信息
    stats = {
        'total_processed': total_books,
        'total_tags': total_tags,
        'generated_files': len(generated_files),
        'elapsed_time': elapsed_time,
        'tags': {tag: len(books) for tag, books in sorted(books_by_tag.items())}
    }
    save_stats(stats)
    
    # 保存已处理的ID
    all_processed = processed_ids | set(book_ids)
    save_processed_ids(all_processed)
    
    # 保存最大书籍ID(用于增量更新)
    if book_ids:
        max_id = max(book_ids)
        save_max_book_id(max_id)
        print(f"\n📊 最大书籍ID: {max_id}(已保存,用于增量更新)")
    
    # 生成热门分类索引文件
    print(f"\n📝 生成热门分类索引文件...")
    hot_categories_file = generate_hot_categories_index(books_by_tag, OUTPUT_DIR)
    print(f"  ✅ 热门分类索引: {hot_categories_file.name}")
    
    print(f"\n📈 统计信息已保存: {STATS_FILE}")
    print("=" * 80)
    print("✅ 测试完成!")
    print("=" * 80)


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='批量处理书籍并生成md文件')
    parser.add_argument('--start-id', type=int, default=1, help='起始书籍ID(默认:1)')
    parser.add_argument('--end-id', type=int, default=1000, help='结束书籍ID(默认:1000)')
    args = parser.parse_args()
    
    asyncio.run(main(args.start_id, args.end_id))