Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Merge branch 'codex/reading-learning-prs-20260923' into codex/pr-batch-root

Bingxi Zhao (Frank)705e9bf9d519
diff --cc deeptutor/reading/extract.py
index d0c29da9e,84eb8c789..8330abb31
--- a/deeptutor/reading/extract.py
+++ b/deeptutor/reading/extract.py
@@@ -98,35 -77,7 +98,35 @@@ class Extraction
return sum(len(u) for u in self.units)
+def _media_for_units(
+ units: tuple[str, ...], images: tuple[EmbeddedImage, ...]
+) -> tuple[MediaItem, ...]:
+ """Map ``[图片 N]`` markers in unit texts to their image bytes.
+
+ A marker names the image it belongs to, so the mapping needs no position
+ bookkeeping: scan each unit, look the name up, and pin the image to that
+ locator. Repeated markers (the same figure floated into two sections) are
+ kept once per locator; unknown names are skipped silently — the text still
+ stands on its own.
+ """
+ by_name = {image.name: image for image in images}
+ if not by_name:
+ return ()
+ items: list[MediaItem] = []
+ seen: set[tuple[int, str]] = set()
+ for locator, unit in enumerate(units, start=1):
+ for _, name in find_markers(unit):
+ image = by_name.get(name)
+ if image is None or (locator, name) in seen:
+ continue
+ seen.add((locator, name))
+ items.append(
+ MediaItem(name=name, mime_type=image.mime_type, data=image.data, locator=locator)
+ )
+ return tuple(items)
+
+
- def extract_material(path: str | Path) -> Extraction:
+ def extract_material(path: str | Path, *, data: bytes | None = None) -> Extraction:
"""Cut *path* into units, dispatching on its extension.
Raises :class:`ReadingError` when the file cannot be read at all, or when
@@@ -196,42 -147,7 +203,42 @@@ def _extract_pdf(source: Path) -> Extra
)
+def _pdf_pages_with_image_markers(
+ units: tuple[str, ...], source: Path
+) -> tuple[tuple[str, ...], tuple[MediaItem, ...]]:
+ """Append the page's ``[图片 N: name]`` markers to that page's text tail.
+
+ Markers are appended only at the very end of each unit so the character
+ offsets of the page's own prose never move — stored per-character
+ annotations resolve against quotes lifted from this text, and shifting them
+ would silently detach every highlight. The markers name the images, which
+ :func:`_media_for_units` then resolves back into locator-pinned media via
+ the same mapping DOCX/PPTX already use.
+
+ Best-effort by contract: any failure or empty result returns the units and
+ no media untouched, so an image quirk never turns a readable PDF into a
+ failed ingest.
+ """
+ try:
+ pdf_images = extract_pdf_images(source.read_bytes(), budget=reading_image_budget())
+ except Exception:
+ logger.warning("%s: PDF image extraction failed", source.name, exc_info=True)
+ return units, ()
+ if not pdf_images.collection.images:
+ return units, ()
+
+ marker_by_page: dict[int, list[str]] = {}
+ for page_number, indices in pdf_images.page_map:
+ marker_by_page[page_number] = [
+ build_marker(pdf_images.collection.images[index]) for index in indices
+ ]
+ targeted = tuple(
+ unit + ("\n" + "\n".join(marker_by_page[i]) if i in marker_by_page else "")
+ for i, unit in enumerate(units, 1)
+ )
+ return targeted, _media_for_units(targeted, pdf_images.collection.images)
+
- def _extract_epub(source: Path) -> Extraction:
+ def _extract_epub(data: bytes, filename: str) -> Extraction:
"""Preserve EPUB spine order so browser and assistant locators agree."""
from deeptutor.utils.document_extractor import DocumentExtractionError, extract_epub_spine
diff --cc deeptutor/reading/store.py
index ac79bc2b2,40494f3ca..b93beb540
--- a/deeptutor/reading/store.py
+++ b/deeptutor/reading/store.py
@@@ -75,8 -74,7 +75,9 @@@ MAX_BOOKMARKS = 20
UNIT_REFS_NAME = "unit_refs.json"
UNITS_DIR = "units"
RAW_DIR = "raw"
+MEDIA_DIR = "media"
+MEDIA_INDEX_NAME = "media.json"
+ RENDER_DIR = "render"
ASSETS_DIR = "assets"
REVISIONS_DIR = "revisions"
@@@ -882,62 -766,13 +935,69 @@@ class ReadingStore
return None
return self._find_raw(self._dir(material_id))
+ def media_items(self, material_id: str) -> list[dict[str, Any]]:
+ """The embedded-image index: name / locator / mime / byte size rows."""
+ rows = _read_json(self._dir(material_id) / MEDIA_INDEX_NAME)
+ if not isinstance(rows, list):
+ return []
+ return [row for row in rows if isinstance(row, dict) and row.get("name")]
+
+ def media_items_at(self, material_id: str, locator: int) -> list[dict[str, Any]]:
+ """Embedded images attached to one locator, in extraction order."""
+ return [row for row in self.media_items(material_id) if row.get("locator") == locator]
+
+ def media_path(self, material_id: str, name: str) -> Path | None:
+ """Resolve a media file by index-validated name (no traversal)."""
+ clean = posixpath.basename(str(name or "").replace("\\", "/"))
+ if not clean:
+ return None
+ if not any(row.get("name") == clean for row in self.media_items(material_id)):
+ return None
+ path = self._dir(material_id) / MEDIA_DIR / clean
+ return path if path.is_file() else None
+
+ def update_media_captions(
+ self, material_id: str, captions: Mapping[str, str]
+ ) -> int:
+ """Write per-image captions into the media index. Returns rows changed.
+
+ *captions* maps an image name to its text. Names absent from the index
+ are ignored silently (extraction may have dropped a figure), and an
+ empty value leaves the existing caption alone rather than erasing it.
+ The index is only rewritten when something actually changed, under the
+ same per-material lock every other write uses.
+ """
+ wanted = {
+ str(name): str(value).strip()
+ for name, value in captions.items()
+ if str(value or "").strip()
+ }
+ if not wanted:
+ return 0
+ index_path = self._dir(material_id) / MEDIA_INDEX_NAME
+ with self._locked(material_id):
+ rows = _read_json(index_path)
+ if not isinstance(rows, list):
+ return 0
+ changed = 0
+ for row in rows:
+ if not isinstance(row, dict):
+ continue
+ name = str(row.get("name") or "")
+ caption = wanted.get(name)
+ if caption and str(row.get("caption") or "") != caption:
+ row["caption"] = caption
+ changed += 1
+ if changed:
+ _atomic_write(index_path, json.dumps(rows, ensure_ascii=False))
+ return changed
+ def render_path(self, material_id: str) -> Path | None:
+ """Browser-ready EPUB archive; original bytes for all other formats."""
+ manifest = self.manifest(material_id)
+ if manifest.render_mode == "text":
+ return None
+ material_dir = self._dir(material_id)
+ return self._find_file_in_dir(material_dir / RENDER_DIR) or self._find_raw(material_dir)
def _has_raw(self, material_id: str) -> bool:
"""Whether original bytes are already on disk, without loading them."""
diff --cc web/locales/en/app.json
index 84f98050c,0d324b89b..9c9f95a9d
--- a/web/locales/en/app.json
+++ b/web/locales/en/app.json
@@@ -3106,19 -3051,6 +3106,14 @@@
"Send to Notebook": "Send to Notebook",
"Build Mastery Path": "Build Mastery Path",
"Add source": "Add source",
- "Linked folders": "Linked folders",
+ "Keep a local folder as a source. Uploads copy files once; links remember the source for manual synchronization.": "Keep a local folder as a source. Uploads copy files once; links remember the source for manual synchronization.",
- "Link folder": "Link folder",
+ "Folder linked.": "Folder linked.",
+ "Folder unlinked.": "Folder unlinked.",
+ "No new or modified files to sync.": "No new or modified files to sync.",
+ "Syncing {{count}} changed file(s).": "Syncing {{count}} changed file(s).",
+ "Sync started for {{count}} changed file(s).": "Sync started for {{count}} changed file(s).",
+ "No linked folders yet. Click \"Link folder\" to keep a local source in sync.": "No linked folders yet. Click \"Link folder\" to keep a local source in sync.",
+ "Last synced": "Last synced",
- "Sync now": "Sync now",
- "Syncing…": "Syncing…",
- "Unlink folder": "Unlink folder",
"Remove source": "Remove source",
"Transcript": "Transcript",
"{{count}} passages available to the companion": "{{count}} passages available to the companion",
@@@ -4934,8 -4875,8 +4929,6 @@@
"Delete file": "Delete file",
"GitHub Sources": "GitHub Sources",
"Track a GitHub repo's Markdown docs. DeepTutor auto-syncs daily; you can also trigger a sync manually.": "Track a GitHub repo's Markdown docs. DeepTutor auto-syncs daily; you can also trigger a sync manually.",
-- "Syncing…": "Syncing…",
-- "Sync now": "Sync now",
"Repo (owner/name)": "Repo (owner/name)",
"Branch": "Branch",
"Path prefix": "Path prefix",
@@@ -5089,46 -5030,20 +5082,69 @@@
"Other — write your own reply…": "Other — write your own reply…",
"Pick an option or type your own to continue.": "Pick an option or type your own to continue.",
"{{count}} questions — tap a tab to switch.": "{{count}} questions — tap a tab to switch.",
+ "settings.presets.title": "Starting points",
+ "settings.presets.blurb": "Load a reviewed preset as a draft. Nothing changes until you apply it.",
+ "settings.presets.error": "Presets could not be loaded.",
+ "settings.presets.empty": "No presets are available.",
+ "settings.presets.loadDraft": "Load draft",
+ "settings.presets.parser": "Parser",
+ "settings.presets.prerequisites": "Needs",
+ "settings.presets.credentials": "Credentials",
+ "settings.presets.features": "Unlocks",
+ "settings.presets.cost.low": "Low cost",
+ "settings.presets.cost.medium": "Medium cost",
+ "settings.presets.cost.high": "High cost",
+ "settings.presets.basic_text.name": "Basic text",
+ "settings.presets.basic_text.description": "A quiet text-first setup for chat, notes, and focused reasoning.",
+ "settings.presets.local_multimodal.name": "Local multimodal",
+ "settings.presets.local_multimodal.description": "Parse documents locally and prepare image and video generation tools.",
+ "settings.presets.university_study.name": "University study",
+ "settings.presets.university_study.description": "Combine document parsing with web and paper search for research work.",
+ "settings.presets.full_generation.name": "Full generation",
+ "settings.presets.full_generation.description": "Turn on the complete generation toolkit when every provider is configured.",
+ "settings.presets.credential.llm": "Language model",
+ "settings.presets.credential.imagegen": "Image model",
+ "settings.presets.credential.videogen": "Video model",
+ "settings.presets.prerequisite.llm_profile": "model profile",
+ "settings.presets.prerequisite.markdown_parser": "markdown parser",
+ "settings.presets.prerequisite.media_profiles": "media profiles",
+ "settings.presets.prerequisite.search_profile": "search profile",
+ "settings.presets.feature.core_chat": "core chat",
+ "settings.presets.feature.reasoning": "reasoning",
+ "settings.presets.feature.document_parsing": "document parsing",
+ "settings.presets.feature.web_search": "web search",
+ "settings.presets.feature.paper_search": "paper search",
+ "settings.presets.feature.image_generation": "image generation",
+ "settings.presets.feature.video_generation": "video generation",
+ "Preset loaded as a draft — review before applying": "Preset loaded as a draft — review before applying",
+ "Could not load the preset: {{message}}": "Could not load the preset: {{message}}",
+ "Public device sign-in": "Public device sign-in",
+ "Public HTTPS origin": "Public HTTPS origin",
+ "Create link": "Create link",
+ "Pairing code": "Pairing code",
+ "Completing sign-in…": "Completing sign-in…",
- "Expires in {{seconds}} seconds": "Expires in {{seconds}} seconds"
++ "Expires in {{seconds}} seconds": "Expires in {{seconds}} seconds",
++ "Enter fullscreen": "Enter fullscreen",
++ "Fullscreen is unavailable for this player.": "Fullscreen is unavailable for this player.",
++ "Not found": "Not found",
++ "Ambiguous": "Ambiguous",
++ "Only chapter markers are available for this video.": "Only chapter markers are available for this video.",
++ "Browse Invidious": "Browse Invidious",
++ "Choose a video to add to this collection.": "Choose a video to add to this collection.",
+ "Learning progress": "Learning progress",
+ "Review your own reading and learning activity.": "Review your own reading and learning activity.",
+ "Unable to load learning progress": "Unable to load learning progress",
+ "Loading progress": "Loading progress",
+ "No learning activity yet": "No learning activity yet",
+ "Learning actions": "Learning actions",
+ "Recent learning actions": "Recent learning actions",
+ "Reading progress": "Reading progress",
+ "No reading progress yet": "No reading progress yet",
+ "Furthest reading progress": "Furthest reading progress",
+ "No learning actions yet": "No learning actions yet",
+ "Locator": "Locator",
+ "Search materials and contents": "Search materials and contents",
+ "No matching materials or contents.": "No matching materials or contents.",
+ "Fullscreen learning": "Fullscreen learning",
+ "Exit learning mode": "Exit learning mode"
}
diff --cc web/locales/zh/app.json
index 77cb53445,7eab9ca01..bdb50d267
--- a/web/locales/zh/app.json
+++ b/web/locales/zh/app.json
@@@ -2162,37 -2151,6 +2162,37 @@@
"Knowledge base root": "知识库根目录",
"Folder name": "文件夹名称",
"Or select an entire folder": "或选择整个文件夹",
+ "Or upload a folder (one-time import)": "或上传文件夹(一次性导入)",
+ "Choosing a folder here is a one-time import. For a folder that stays in sync, use Linked folders.": "在此选择文件夹只会执行一次性导入。若要持续同步,请使用“关联文件夹”。",
+ "Linked folders": "关联文件夹",
+ "Link a local folder": "关联本地文件夹",
+ "Link folder": "关联文件夹",
+ "Linking…": "正在关联…",
+ "Folder path is required.": "请输入文件夹路径。",
+ "e.g. /Users/name/Documents or ~/notes": "例如 /Users/name/Documents 或 ~/notes",
+ "Use a folder path on the machine running DeepTutor. Linking records the source; files are not imported until you click Sync now.": "请输入运行 DeepTutor 的机器上的文件夹路径。关联只会记录来源;点击“立即同步”前不会导入文件。",
+ "Keep a local folder as a document source and sync new or modified supported files when you choose.": "将本地文件夹作为文档来源,并在需要时同步新增或修改过的受支持文件。",
+ "No linked folders yet. Click \"Link folder\" to add a source.": "暂未关联文件夹。点击“关联文件夹”添加来源。",
+ "{{count}} tracked files": "已跟踪 {{count}} 个文件",
+ "Linked {{time}}": "关联于 {{time}}",
+ "Last successful sync": "上次成功同步",
+ "Never synced": "从未同步",
+ "Check this folder for new or modified files": "检查此文件夹中的新增或修改文件",
- "Unlink folder": "解除关联",
++ "Unlink folder": "解除文件夹关联",
+ "Unlink {{path}}": "解除关联 {{path}}",
+ "Unlinking stops future syncs. Files already imported remain in this knowledge base.": "解除关联后将停止后续同步。已导入此知识库的文件会保留。",
+ "Sync now imports new or modified supported files. Deleted or renamed source files are not removed yet.": "立即同步会导入新增或修改过的受支持文件。已删除或重命名的源文件暂不会被移除。",
+ "Sync Process": "同步过程",
+ "Sync linked folder": "同步关联文件夹",
+ "Sync now": "立即同步",
+ "Syncing…": "正在同步…",
+ "Queued {{count}} files for indexing.": "已将 {{count}} 个文件加入索引队列。",
+ "{{new}} new, {{modified}} modified.": "新增 {{new}} 个,修改 {{modified}} 个。",
+ "Loading linked folders": "正在加载关联文件夹",
+ "Timed out loading linked folders. Click retry.": "加载关联文件夹超时,请点击重试。",
+ "This knowledge base is read-only. Linked folders can be viewed but not changed.": "此知识库为只读。可以查看关联文件夹,但不能修改。",
+ "Assigned knowledge bases are read-only.": "管理员分配的知识库为只读。",
+ "Wait for the current sync to finish.": "请等待当前同步完成。",
"Unsupported files are skipped; the rest will be indexed.": "不支持的文件将被跳过,其余照常索引。",
"{{ready}} ready, {{skip}} will be skipped": "{{ready}} 个就绪,{{skip}} 个将跳过",
"Allow automatic model download": "允许自动下载模型",
@@@ -3106,19 -3051,6 +3106,14 @@@
"Send to Notebook": "发送到 Notebook",
"Build Mastery Path": "生成精通之路",
"Add source": "添加资料",
- "Linked folders": "关联文件夹",
+ "Keep a local folder as a source. Uploads copy files once; links remember the source for manual synchronization.": "将本地文件夹保留为资料来源。上传只复制一次;关联会记住来源,便于手动同步。",
- "Link folder": "关联文件夹",
+ "Folder linked.": "文件夹已关联。",
+ "Folder unlinked.": "文件夹已解除关联。",
+ "No new or modified files to sync.": "没有需要同步的新文件或修改。",
+ "Syncing {{count}} changed file(s).": "正在同步 {{count}} 个有变化的文件。",
+ "Sync started for {{count}} changed file(s).": "已开始同步 {{count}} 个有变化的文件。",
+ "No linked folders yet. Click \"Link folder\" to keep a local source in sync.": "还没有关联文件夹。点击「关联文件夹」,让本地来源保持同步。",
+ "Last synced": "上次同步",
- "Sync now": "立即同步",
- "Syncing…": "正在同步…",
- "Unlink folder": "解除文件夹关联",
"Remove source": "移除资料",
"Transcript": "转写文本",
"{{count}} passages available to the companion": "AI 伙伴可读取 {{count}} 个段落",
@@@ -4934,8 -4875,8 +4929,6 @@@
"Delete file": "删除文件",
"GitHub Sources": "GitHub 来源",
"Track a GitHub repo's Markdown docs. DeepTutor auto-syncs daily; you can also trigger a sync manually.": "跟踪 GitHub 仓库中的 Markdown 文档。DeepTutor 每天自动同步,你也可以手动触发同步。",
-- "Syncing…": "正在同步…",
-- "Sync now": "立即同步",
"Repo (owner/name)": "仓库(所有者/名称)",
"Branch": "分支",
"Path prefix": "路径前缀",
@@@ -5089,46 -5030,20 +5082,69 @@@
"Other — write your own reply…": "其他 — 自定义回复…",
"Pick an option or type your own to continue.": "选择一个选项或输入自定义回复以继续。",
"{{count}} questions — tap a tab to switch.": "{{count}} 个问题 — 点击标签切换。",
+ "settings.presets.title": "配置起点",
+ "settings.presets.blurb": "将预设加载为草稿;只有应用后才会更改设置。",
+ "settings.presets.error": "无法加载配置预设。",
+ "settings.presets.empty": "当前没有可用预设。",
+ "settings.presets.loadDraft": "加载草稿",
+ "settings.presets.parser": "解析引擎",
+ "settings.presets.prerequisites": "前置条件",
+ "settings.presets.credentials": "所需凭据",
+ "settings.presets.features": "解锁能力",
+ "settings.presets.cost.low": "低成本",
+ "settings.presets.cost.medium": "中等成本",
+ "settings.presets.cost.high": "高成本",
+ "settings.presets.basic_text.name": "基础文本",
+ "settings.presets.basic_text.description": "面向聊天、笔记和专注推理的轻量文本配置。",
+ "settings.presets.local_multimodal.name": "本地多模态",
+ "settings.presets.local_multimodal.description": "本地解析文档,并准备图像与视频生成工具。",
+ "settings.presets.university_study.name": "大学学习",
+ "settings.presets.university_study.description": "结合文档解析、网页搜索和论文检索,支持研究型学习。",
+ "settings.presets.full_generation.name": "完整生成",
+ "settings.presets.full_generation.description": "在所有服务配置完成后启用完整生成工具。",
+ "settings.presets.credential.llm": "语言模型",
+ "settings.presets.credential.imagegen": "图像模型",
+ "settings.presets.credential.videogen": "视频模型",
+ "settings.presets.prerequisite.llm_profile": "模型配置",
+ "settings.presets.prerequisite.markdown_parser": "Markdown 解析器",
+ "settings.presets.prerequisite.media_profiles": "媒体服务配置",
+ "settings.presets.prerequisite.search_profile": "搜索服务配置",
+ "settings.presets.feature.core_chat": "核心聊天",
+ "settings.presets.feature.reasoning": "深度推理",
+ "settings.presets.feature.document_parsing": "文档解析",
+ "settings.presets.feature.web_search": "网页搜索",
+ "settings.presets.feature.paper_search": "论文检索",
+ "settings.presets.feature.image_generation": "图像生成",
+ "settings.presets.feature.video_generation": "视频生成",
+ "Preset loaded as a draft — review before applying": "预设已加载为草稿——请检查后再应用",
+ "Could not load the preset: {{message}}": "无法加载预设:{{message}}",
+ "Public device sign-in": "公共设备登录",
+ "Public HTTPS origin": "公共 HTTPS 地址",
+ "Create link": "创建链接",
+ "Pairing code": "配对码",
+ "Completing sign-in…": "正在完成登录…",
- "Expires in {{seconds}} seconds": "{{seconds}} 秒后过期"
++ "Expires in {{seconds}} seconds": "{{seconds}} 秒后过期",
++ "Enter fullscreen": "进入全屏",
++ "Fullscreen is unavailable for this player.": "此播放器不支持全屏。",
++ "Not found": "未找到",
++ "Ambiguous": "位置不明",
++ "Only chapter markers are available for this video.": "这个视频只有章节标记可用。",
++ "Browse Invidious": "浏览 Invidious",
++ "Choose a video to add to this collection.": "选择要添加到这个资料组的视频。",
+ "Learning progress": "学习进度",
+ "Review your own reading and learning activity.": "查看你自己的阅读与学习活动。",
+ "Unable to load learning progress": "无法加载学习进度",
+ "Loading progress": "正在加载进度",
+ "No learning activity yet": "还没有学习活动",
+ "Learning actions": "学习操作",
+ "Recent learning actions": "近期学习操作",
+ "Reading progress": "阅读进度",
+ "No reading progress yet": "还没有阅读进度",
+ "Furthest reading progress": "最远阅读进度",
+ "No learning actions yet": "还没有学习操作",
+ "Locator": "位置",
+ "Search materials and contents": "搜索资料和目录",
+ "No matching materials or contents.": "没有匹配的资料或目录。",
+ "Fullscreen learning": "全屏学习",
+ "Exit learning mode": "退出学习模式"
}