LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
fix: harden MinerU PDF parsing
nash_sudf3d15fde9f4
diff --git a/src/components/settings/sections/mineru-section.tsx b/src/components/settings/sections/mineru-section.tsxindex 9ac9474a..638295a9 100644--- a/src/components/settings/sections/mineru-section.tsx+++ b/src/components/settings/sections/mineru-section.tsx@@ -36,14 +36,20 @@ export function MineruSection({ draft, setDraft }: Props) {<div className="space-y-6"><div><h2 className="text-xl font-semibold">- {t("settings.mineru", { defaultValue: "MinerU PDF Parser" })}+ {t("settings.sections.mineru.title", { defaultValue: "MinerU PDF Parser" })}</h2><p className="text-sm text-muted-foreground mt-1">- {t("settings.mineruDescription", {+ {t("settings.sections.mineru.description", {defaultValue:"Use MinerU cloud API for higher quality PDF parsing (tables, formulas, complex layouts)",})}</p>+ <p className="mt-2 rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-800 dark:text-amber-200">+ {t("settings.sections.mineru.privacyNotice", {+ defaultValue:+ "When enabled, PDF contents are uploaded to MinerU cloud for parsing. Do not enable this for sensitive documents unless you accept that.",+ })}+ </p></div><label className="flex items-center gap-2">@@ -54,7 +60,7 @@ export function MineruSection({ draft, setDraft }: Props) {className="h-4 w-4"/><span className="text-sm">- {t("settings.mineruEnabled", { defaultValue: "Enable MinerU" })}+ {t("settings.sections.mineru.enabled", { defaultValue: "Enable MinerU" })}</span></label>@@ -62,7 +68,7 @@ export function MineruSection({ draft, setDraft }: Props) {<div className="space-y-4 pl-1"><div className="space-y-2"><Label htmlFor="mineru-token">- {t("settings.mineruToken", { defaultValue: "API Token" })}+ {t("settings.sections.mineru.token", { defaultValue: "API Token" })}</Label><Inputid="mineru-token"@@ -72,7 +78,7 @@ export function MineruSection({ draft, setDraft }: Props) {setDraft("mineruToken", e.target.value)setTestState("idle")}}- placeholder={t("settings.mineruTokenHint", {+ placeholder={t("settings.sections.mineru.tokenHint", {defaultValue: "Get your token from mineru.net",})}/>@@ -80,7 +86,7 @@ export function MineruSection({ draft, setDraft }: Props) {<div className="space-y-2"><Label htmlFor="mineru-model">- {t("settings.mineruModel", { defaultValue: "Model Version" })}+ {t("settings.sections.mineru.model", { defaultValue: "Model Version" })}</Label><selectid="mineru-model"@@ -91,22 +97,28 @@ export function MineruSection({ draft, setDraft }: Props) {}><option value="vlm">- {t("settings.mineruModelVlm", {+ {t("settings.sections.mineru.modelVlm", {defaultValue: "VLM (Recommended, best for complex layouts)",})}</option><option value="pipeline">- {t("settings.mineruModelPipeline", {+ {t("settings.sections.mineru.modelPipeline", {defaultValue: "Pipeline (Faster)",})}</option>- <option value="mineru-html">- {t("settings.mineruModelHtml", {- defaultValue: "MinerU-HTML",- })}- </option></select>+ <p className="text-xs text-muted-foreground">+ {t("settings.sections.mineru.modelHint", {+ defaultValue: "PDF parsing supports pipeline and vlm. MinerU-HTML is for HTML files and is not used here.",+ })}+ </p></div>+ <p className="rounded-md bg-muted/50 px-3 py-2 text-xs text-muted-foreground">+ {t("settings.sections.mineru.testQuotaNotice", {+ defaultValue:+ "Connection test submits a small demo file to MinerU and may consume a small amount of quota.",+ })}+ </p><div className="flex items-center gap-3"><Button@@ -118,21 +130,21 @@ export function MineruSection({ draft, setDraft }: Props) {}>{testState === "running"- ? t("settings.mineruTesting", { defaultValue: "Testing..." })- : t("settings.mineruTest", {+ ? t("settings.sections.mineru.testing", { defaultValue: "Testing..." })+ : t("settings.sections.mineru.test", {defaultValue: "Test Connection",})}</Button>{testState === "success" && (<span className="text-sm text-green-600">- {t("settings.mineruTestSuccess", {+ {t("settings.sections.mineru.testSuccess", {defaultValue: "Connection successful",})}</span>)}{testState === "failed" && (<span className="text-sm text-red-600">- {t("settings.mineruTestFailed", {+ {t("settings.sections.mineru.testFailed", {defaultValue: "Test failed",})}: {testError}diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsxindex 4b42cbe6..921c77cd 100644--- a/src/components/settings/settings-view.tsx+++ b/src/components/settings/settings-view.tsx@@ -271,6 +271,7 @@ export function SettingsView() {proxyConfig,scheduledImportConfig,sourceWatchConfig,+ mineruConfig,apiConfig,generalConfig,maxHistoryMessages,@@ -287,6 +288,7 @@ export function SettingsView() {proxyConfig,scheduledImportConfig,sourceWatchConfig,+ mineruConfig,apiConfig,generalConfig,maxHistoryMessages,diff --git a/src/i18n/en.json b/src/i18n/en.jsonindex 3cef23e5..1100f5a2 100644--- a/src/i18n/en.json+++ b/src/i18n/en.json@@ -583,14 +583,16 @@"mineru": {"title": "MinerU PDF Parser","description": "Use MinerU cloud API for higher quality PDF parsing (tables, formulas, complex layouts). Default uses built-in pdfium extraction.",+ "privacyNotice": "When enabled, PDF contents are uploaded to MinerU cloud for parsing. Do not enable this for sensitive documents unless you accept that.","enabled": "Enable MinerU","token": "API Token","tokenHint": "Get your token from mineru.net","model": "Model Version","modelVlm": "VLM (Recommended, best for complex layouts)","modelPipeline": "Pipeline (Faster)",- "modelHtml": "MinerU-HTML",+ "modelHint": "PDF parsing supports pipeline and vlm. MinerU-HTML is for HTML files and is not used here.","test": "Test Connection",+ "testQuotaNotice": "Connection test submits a small demo file to MinerU and may consume a small amount of quota.","testing": "Testing...","testSuccess": "Connection successful","testFailed": "Test failed"diff --git a/src/i18n/zh.json b/src/i18n/zh.jsonindex 7a0eb001..c61862da 100644--- a/src/i18n/zh.json+++ b/src/i18n/zh.json@@ -583,14 +583,16 @@"mineru": {"title": "MinerU PDF 解析","description": "使用 MinerU 云端 API 获得更高质量的 PDF 解析(表格、公式、复杂排版)。默认使用内置 pdfium 提取。",+ "privacyNotice": "启用后,PDF 内容会上传到 MinerU 云端进行解析。除非你接受这一点,否则不要用于敏感文档。","enabled": "启用 MinerU","token": "API Token","tokenHint": "从 mineru.net 获取您的 Token","model": "模型版本","modelVlm": "VLM(推荐,适合复杂排版)","modelPipeline": "Pipeline(更快)",- "modelHtml": "MinerU-HTML",+ "modelHint": "PDF 解析仅使用 pipeline 或 vlm。MinerU-HTML 用于 HTML 文件,这里不会使用。","test": "测试连接",+ "testQuotaNotice": "连接测试会向 MinerU 提交一个小型示例文件,可能消耗少量额度。","testing": "测试中...","testSuccess": "连接成功","testFailed": "测试失败"diff --git a/src/lib/ingest-source-path-collision.test.ts b/src/lib/ingest-source-path-collision.test.tsindex 99d5115c..b293d9ab 100644--- a/src/lib/ingest-source-path-collision.test.ts+++ b/src/lib/ingest-source-path-collision.test.ts@@ -108,10 +108,16 @@ vi.mock("./llm-client", () => ({}),}))+vi.mock("./mineru", () => ({+ parseWithMineru: vi.fn(),+}))+import { autoIngest, executeIngestWrites } from "./ingest"import { streamChat } from "./llm-client"+import { parseWithMineru } from "./mineru"const mockStreamChat = vi.mocked(streamChat)+const mockParseWithMineru = vi.mocked(parseWithMineru)describe("autoIngest source summary paths", () => {let tmp: { path: string; cleanup: () => Promise<void> } | undefined@@ -123,6 +129,7 @@ describe("autoIngest source summary paths", () => {generationSuffix = ""abortDuringReview = nullmockStreamChat.mockClear()+ mockParseWithMineru.mockReset()tmp = await createTempProject("same-basename-sources")await writeFileRaw(`${tmp.path}/purpose.md`, "# Purpose\n\nTrack project config files.\n")@@ -483,6 +490,69 @@ describe("autoIngest source summary paths", () => {).rejects.toThrow("AbortError")})+ it("falls back to built-in PDF extraction when MinerU fails for a non-cancelled ingest", async () => {+ if (!tmp) throw new Error("missing temp project")+ sourceMarkers = ["mineru fallback source"]+ await writeFileRaw(`${tmp.path}/raw/sources/project-a/report.pdf`, "pdf fallback text\n")+ useWikiStore.setState({+ mineruConfig: {+ enabled: true,+ token: "mineru-token",+ modelVersion: "vlm",+ },+ })+ mockParseWithMineru.mockRejectedValueOnce(new Error("network failure from MinerU"))+ const updateSpy = vi.spyOn(useActivityStore.getState(), "updateItem")++ const written = await autoIngest(+ tmp.path,+ `${tmp.path}/raw/sources/project-a/report.pdf`,+ useWikiStore.getState().llmConfig,+ undefined,+ "project-a",+ )++ expect(written.length).toBeGreaterThan(0)+ expect(mockParseWithMineru).toHaveBeenCalled()+ expect(+ updateSpy.mock.calls.some(([, updates]) =>+ updates.detail?.includes("falling back to built-in PDF extraction"),+ ),+ ).toBe(true)+ updateSpy.mockRestore()+ })++ it("does not fall back to built-in PDF extraction when MinerU is cancelled", async () => {+ if (!tmp) throw new Error("missing temp project")+ await writeFileRaw(`${tmp.path}/raw/sources/project-a/cancelled.pdf`, "pdf fallback text\n")+ useWikiStore.setState({+ mineruConfig: {+ enabled: true,+ token: "mineru-token",+ modelVersion: "vlm",+ },+ })+ const controller = new AbortController()+ controller.abort()+ mockParseWithMineru.mockRejectedValueOnce(new Error("MinerU parsing cancelled"))++ await expect(+ autoIngest(+ tmp.path,+ `${tmp.path}/raw/sources/project-a/cancelled.pdf`,+ useWikiStore.getState().llmConfig,+ controller.signal,+ "project-a",+ ),+ ).rejects.toThrow("MinerU parsing cancelled")++ expect(+ useActivityStore.getState().items.some((item) =>+ item.detail?.includes("falling back to built-in PDF extraction"),+ ),+ ).toBe(false)+ })+it("canonicalizes interactive source summary paths and sources frontmatter", async () => {if (!tmp) throw new Error("missing temp project")diff --git a/src/lib/ingest.ts b/src/lib/ingest.tsindex c4ec78ab..8393ce84 100644--- a/src/lib/ingest.ts+++ b/src/lib/ingest.ts@@ -483,6 +483,7 @@ async function autoIngestImpl(const isPdf = lowerExt === "pdf"const mineruCfg = useWikiStore.getState().mineruConfigif (isPdf && mineruCfg.enabled && mineruCfg.token) {+ let mineruSucceeded = falsetry {const cacheDir = sp.substring(0, sp.lastIndexOf("/"))const cachePath = `${cacheDir}/.cache/${fileName}.txt`@@ -490,15 +491,22 @@ async function autoIngestImpl(console.log(`[ingest:mineru] submitting "${fileName}" to MinerU API`)const markdown = await parseWithMineru(mineruCfg, sp, undefined, (msg) => {activity.updateItem(activityId, { detail: `MinerU: ${msg}` })- })+ }, signal)await createDirectory(`${cacheDir}/.cache`)await writeFile(cachePath, markdown)+ mineruSucceeded = trueconsole.log(`[ingest:mineru] cached MinerU output for "${fileName}" (${markdown.length} chars)`)} catch (err) {- const msg = err instanceof Error ? err.message : String(err)+ if (signal?.aborted) throw err+ const msg = trimInlineStatus(err instanceof Error ? err.message : String(err))console.warn(`[ingest:mineru] MinerU parsing failed, falling back to pdfium: ${msg}`)+ activity.updateItem(activityId, {+ detail: `MinerU failed, falling back to built-in PDF extraction: ${msg}`,+ })+ }+ if (mineruSucceeded && !signal?.aborted) {+ activity.updateItem(activityId, { detail: "Reading source..." })}- activity.updateItem(activityId, { detail: "Reading source..." })}const [sourceContent, schema, purpose, index, overview] = await Promise.all([@@ -1976,6 +1984,10 @@ function trimLongText(text: string, maxChars: number): string {return `${text.slice(0, maxChars).trimEnd()}\n\n[...trimmed for prompt budget...]`}+function trimInlineStatus(text: string, maxChars = 240): string {+ return text.length <= maxChars ? text : `${text.slice(0, maxChars).trimEnd()}...`+}+function hashTextHex(text: string): string {// 64-bit FNV-1a over UTF-16 code units. This is a stability key, not// a security primitive; validation also checks source length/chunkdiff --git a/src/lib/mineru.test.ts b/src/lib/mineru.test.tsnew file mode 100644index 00000000..e3f4a73f--- /dev/null+++ b/src/lib/mineru.test.ts@@ -0,0 +1,333 @@+import JSZip from "jszip"+import { beforeEach, describe, expect, it, vi } from "vitest"++const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()++vi.mock("@/lib/tauri-fetch", () => ({+ getHttpFetch: () => Promise.resolve(mockHttpFetch),+}))++const fsMocks = vi.hoisted(() => ({+ getFileSize: vi.fn<() => Promise<number>>(),+ readFileAsBase64: vi.fn<() => Promise<{ base64: string; mimeType: string }>>(),+}))++vi.mock("@/commands/fs", () => ({+ getFileSize: fsMocks.getFileSize,+ readFileAsBase64: fsMocks.readFileAsBase64,+}))++import { __mineruTest, parseWithMineru, testMineruConnection } from "./mineru"++function jsonResponse(body: unknown, init?: ResponseInit): Response {+ return new Response(JSON.stringify(body), {+ status: 200,+ headers: { "Content-Type": "application/json" },+ ...init,+ })+}++async function zipResponse(files: Record<string, string>): Promise<Response> {+ const zip = new JSZip()+ for (const [path, content] of Object.entries(files)) {+ zip.file(path, content)+ }+ const bytes = await zip.generateAsync({ type: "uint8array" })+ const buffer = bytes.buffer.slice(+ bytes.byteOffset,+ bytes.byteOffset + bytes.byteLength,+ ) as ArrayBuffer+ return new Response(buffer)+}++beforeEach(() => {+ mockHttpFetch.mockReset()+ fsMocks.getFileSize.mockReset()+ fsMocks.readFileAsBase64.mockReset()+ fsMocks.getFileSize.mockResolvedValue(1024)+ fsMocks.readFileAsBase64.mockResolvedValue({+ base64: btoa("pdf bytes"),+ mimeType: "application/pdf",+ })+})++describe("MinerU API helpers", () => {+ it("maps official API error codes to actionable messages", () => {+ expect(__mineruTest.mineruApiErrorMessage("A0202", "bad token")).toContain("invalid")+ expect(__mineruTest.mineruApiErrorMessage("A0211", "expired")).toContain("expired")+ expect(__mineruTest.mineruApiErrorMessage(-60005, "too large")).toContain("200 MB")+ expect(__mineruTest.mineruApiErrorMessage(-60006, "too many pages")).toContain("200 page")+ expect(__mineruTest.mineruApiErrorMessage(-60018, "quota")).toContain("quota")+ expect(__mineruTest.mineruApiErrorMessage(123, "other")).toBe("MinerU API error 123: other")+ })++ it("prefers full.md from MinerU result zip", async () => {+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({+ "result/other.md": "other markdown",+ "result/full.md": "full markdown",+ }))++ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))+ .resolves.toBe("full markdown")+ })++ it("falls back to another markdown file when full.md is missing", async () => {+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({+ "result/page.md": "fallback markdown",+ }))++ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))+ .resolves.toBe("fallback markdown")+ })++ it("rejects MinerU zip files without markdown output", async () => {+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({+ "result/layout.json": "{}",+ }))++ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))+ .rejects.toThrow("No Markdown file")+ })+})++describe("parseWithMineru", () => {+ it("rejects unsupported MinerU model versions before reading or uploading", async () => {+ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "mineru-html" as "vlm",+ }, "/tmp/doc.pdf")).rejects.toThrow("pipeline or vlm")++ expect(fsMocks.getFileSize).not.toHaveBeenCalled()+ expect(mockHttpFetch).not.toHaveBeenCalled()+ })++ it("rejects local files over MinerU's 200 MB accurate parsing limit before upload", async () => {+ fsMocks.getFileSize.mockResolvedValue(__mineruTest.MAX_ACCURATE_PARSE_BYTES + 1)++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/large.pdf")).rejects.toThrow("200 MB")++ expect(mockHttpFetch).not.toHaveBeenCalled()+ expect(fsMocks.readFileAsBase64).not.toHaveBeenCalled()+ })++ it("rejects before network access when the abort signal is already cancelled", async () => {+ const controller = new AbortController()+ controller.abort()++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf", undefined, undefined, controller.signal)).rejects.toThrow("cancelled")++ expect(mockHttpFetch).not.toHaveBeenCalled()+ })++ it("rejects batch upload responses without an upload URL", async () => {+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", file_urls: [] },+ }))++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf")).rejects.toThrow("upload URL")+ })++ it("uploads the decoded PDF bytes to the MinerU upload URL", async () => {+ fsMocks.readFileAsBase64.mockResolvedValueOnce({+ base64: btoa("custom pdf bytes"),+ mimeType: "application/pdf",+ })+ mockHttpFetch+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },+ }))+ .mockResolvedValueOnce(new Response("", { status: 200 }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: {+ batch_id: "batch-1",+ extract_result: [{ file_name: "doc.pdf", state: "done", full_zip_url: "https://zip" }],+ },+ }))+ .mockResolvedValueOnce(await zipResponse({ "full.md": "parsed markdown" }))++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf")).resolves.toBe("parsed markdown")++ const uploadBody = mockHttpFetch.mock.calls[1]?.[1]?.body+ expect(uploadBody).toBeInstanceOf(ArrayBuffer)+ expect(new TextDecoder().decode(uploadBody as ArrayBuffer)).toBe("custom pdf bytes")+ })++ it("submits URL tasks without reading or uploading a local file", async () => {+ mockHttpFetch+ .mockResolvedValueOnce(jsonResponse({+ code: "0",+ msg: "ok",+ data: { task_id: "task-1" },+ }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { task_id: "task-1", state: "done", full_zip_url: "https://zip" },+ }))+ .mockResolvedValueOnce(await zipResponse({ "full.md": "url markdown" }))++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "pipeline",+ }, "/tmp/doc.pdf", "https://example.test/doc.pdf")).resolves.toBe("url markdown")++ expect(fsMocks.getFileSize).not.toHaveBeenCalled()+ expect(fsMocks.readFileAsBase64).not.toHaveBeenCalled()+ expect(JSON.parse(String(mockHttpFetch.mock.calls[0]?.[1]?.body))).toMatchObject({+ url: "https://example.test/doc.pdf",+ model_version: "pipeline",+ })+ })++ it("rejects MinerU failed states with the service error message", async () => {+ mockHttpFetch+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },+ }))+ .mockResolvedValueOnce(new Response("", { status: 200 }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: {+ batch_id: "batch-1",+ extract_result: [{ file_name: "doc.pdf", state: "failed", err_msg: "parse exploded" }],+ },+ }))++ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf")).rejects.toThrow("parse exploded")+ })++ it("stops polling immediately when the abort signal fires during the poll interval", async () => {+ mockHttpFetch+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },+ }))+ .mockResolvedValueOnce(new Response("", { status: 200 }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "running" }] },+ }))++ const controller = new AbortController()+ const result = parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf", undefined, undefined, controller.signal)++ setTimeout(() => controller.abort(), 10)++ await expect(result).rejects.toThrow("cancelled")+ expect(mockHttpFetch).toHaveBeenCalledTimes(3)+ })++ it("handles official pending, waiting-file, converting, and running states before completion", async () => {+ mockHttpFetch+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },+ }))+ .mockResolvedValueOnce(new Response("", { status: 200 }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "pending" }] },+ }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "waiting-file" }] },+ }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "converting" }] },+ }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "running" }] },+ }))+ .mockResolvedValueOnce(jsonResponse({+ code: 0,+ msg: "ok",+ data: {+ batch_id: "batch-1",+ extract_result: [{ file_name: "doc.pdf", state: "done", full_zip_url: "https://zip" }],+ },+ }))+ .mockResolvedValueOnce(await zipResponse({ "full.md": "parsed markdown" }))++ const progress: string[] = []+ await expect(parseWithMineru({+ enabled: true,+ token: "token",+ modelVersion: "vlm",+ }, "/tmp/doc.pdf", undefined, (msg) => progress.push(msg))).resolves.toBe("parsed markdown")++ expect(progress).toContain("Waiting for MinerU to finish...")+ expect(mockHttpFetch).toHaveBeenCalledTimes(8)+ }, 16_000)+})++describe("testMineruConnection", () => {+ it("resolves when MinerU accepts the connection test task", async () => {+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({+ code: "0",+ msg: "ok",+ data: { task_id: "task-1" },+ }))++ await expect(testMineruConnection("token")).resolves.toBeUndefined()+ })++ it("includes HTTP status and response body when connection test transport fails", async () => {+ mockHttpFetch.mockResolvedValueOnce(new Response("bad gateway", { status: 502 }))++ await expect(testMineruConnection("token")).rejects.toThrow("HTTP 502: bad gateway")+ })++ it("maps MinerU API errors during connection test", async () => {+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({+ code: "A0202",+ msg: "token invalid",+ data: {},+ }))++ await expect(testMineruConnection("bad-token")).rejects.toThrow("invalid")+ })+})diff --git a/src/lib/mineru.ts b/src/lib/mineru.tsindex 75c02e1e..f72f9002 100644--- a/src/lib/mineru.ts+++ b/src/lib/mineru.ts@@ -1,25 +1,28 @@import JSZip from "jszip"import type { MineruConfig } from "@/stores/wiki-store"-import { readFileAsBase64 } from "@/commands/fs"+import { getFileSize, readFileAsBase64 } from "@/commands/fs"import { getHttpFetch } from "@/lib/tauri-fetch"const API_BASE = "https://mineru.net/api/v4"const POLL_INTERVAL_MS = 3_000const POLL_TIMEOUT_MS = 300_000 // 5 minutes+const MAX_ACCURATE_PARSE_BYTES = 200 * 1024 * 1024// ── Types ──interface TaskResponse {- code: number+ code: number | stringdata: { task_id: string }msg: string}+type MineruTaskState = "pending" | "running" | "converting" | "done" | "failed" | "waiting-file"+interface TaskStatus {- code: number+ code: number | stringdata: {task_id: string- state: "running" | "done" | "failed"+ state: MineruTaskStatefull_zip_url?: stringerr_msg?: stringextract_progress?: { extracted_pages: number; total_pages: number }@@ -28,12 +31,12 @@ interface TaskStatus {}interface BatchStatus {- code: number+ code: number | stringdata: {batch_id: stringextract_result: Array<{file_name: string- state: "running" | "done" | "failed"+ state: MineruTaskStatefull_zip_url?: stringerr_msg?: string}>@@ -42,7 +45,7 @@ interface BatchStatus {}interface UploadUrlResponse {- code: number+ code: number | stringdata: {batch_id: stringfile_urls: string[]@@ -59,20 +62,71 @@ async function mineruHeaders(token: string): Promise<HeadersInit> {}}+function mineruApiErrorMessage(code: number | string | undefined, msg?: string): string {+ const key = String(code ?? "")+ const known: Record<string, string> = {+ A0202: "MinerU token is invalid. Check the API token in Settings.",+ A0211: "MinerU token has expired. Create a new API token and update Settings.",+ "-60005": "MinerU rejected the file because it is larger than 200 MB.",+ "-60006": "MinerU rejected the file because it exceeds the 200 page limit.",+ "-60018": "MinerU daily parsing quota has been reached.",+ }+ const knownMessage = known[key]+ if (knownMessage) return msg ? `${knownMessage} (${msg})` : knownMessage+ return msg ? `MinerU API error ${key || "unknown"}: ${msg}` : `MinerU API error ${key || "unknown"}`+}++function assertMineruSuccess(json: { code: number | string; msg?: string }): void {+ if (json.code !== 0 && json.code !== "0") {+ throw new Error(mineruApiErrorMessage(json.code, json.msg))+ }+}++function throwIfAborted(signal?: AbortSignal): void {+ if (signal?.aborted) {+ throw new Error("MinerU parsing cancelled")+ }+}++function decodeBase64ToBytes(base64: string): Uint8Array {+ if (typeof atob !== "function") {+ throw new Error("Base64 decoding is not available in this runtime")+ }+ const binaryStr = atob(base64)+ const bytes = new Uint8Array(binaryStr.length)+ for (let i = 0; i < binaryStr.length; i++) {+ bytes[i] = binaryStr.charCodeAt(i)+ }+ return bytes+}++function bytesToUploadBody(bytes: Uint8Array): ArrayBuffer {+ if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {+ return bytes.buffer as ArrayBuffer+ }+ return bytes.buffer.slice(+ bytes.byteOffset,+ bytes.byteOffset + bytes.byteLength,+ ) as ArrayBuffer+}+async function submitUrlTask(token: string,url: string,modelVersion: string,+ signal?: AbortSignal,): Promise<string> {const httpFetch = await getHttpFetch()+ throwIfAborted(signal)const res = await httpFetch(`${API_BASE}/extract/task`, {method: "POST",headers: await mineruHeaders(token),+ signal,body: JSON.stringify({ url, model_version: modelVersion }),})if (!res.ok) throw new Error(`MinerU submit failed: HTTP ${res.status}`)const json: TaskResponse = await res.json()- if (json.code !== 0) throw new Error(`MinerU API error: ${json.msg}`)+ assertMineruSuccess(json)return json.data.task_id}@@ -81,14 +135,17 @@ async function uploadFileForTask(fileName: string,fileBase64: string,modelVersion: string,+ signal?: AbortSignal,): Promise<{ batchId: string; uploadUrl: string }> {const httpFetch = await getHttpFetch()const headers = await mineruHeaders(token)+ throwIfAborted(signal)// Step 1: Get upload URLconst res = await httpFetch(`${API_BASE}/file-urls/batch`, {method: "POST",headers,+ signal,body: JSON.stringify({files: [{ name: fileName, data_id: fileName }],model_version: modelVersion,@@ -96,21 +153,22 @@ async function uploadFileForTask(})if (!res.ok) throw new Error(`MinerU batch submit failed: HTTP ${res.status}`)const json: UploadUrlResponse = await res.json()- if (json.code !== 0) throw new Error(`MinerU API error: ${json.msg}`)+ assertMineruSuccess(json)const batchId = json.data.batch_idconst uploadUrl = json.data.file_urls[0]+ if (!batchId || !uploadUrl) {+ throw new Error("MinerU did not return a file upload URL")+ }// Step 2: Upload file binary (convert base64 back to binary)- const binaryStr = atob(fileBase64)- const bytes = new Uint8Array(binaryStr.length)- for (let i = 0; i < binaryStr.length; i++) {- bytes[i] = binaryStr.charCodeAt(i)- }+ const bytes = decodeBase64ToBytes(fileBase64)+ throwIfAborted(signal)const uploadRes = await httpFetch(uploadUrl, {method: "PUT",- body: bytes,+ signal,+ body: bytesToUploadBody(bytes),})if (!uploadRes.ok && uploadRes.status !== 200 && uploadRes.status !== 201) {throw new Error(`MinerU file upload failed: HTTP ${uploadRes.status}`)@@ -119,17 +177,36 @@ async function uploadFileForTask(return { batchId, uploadUrl }}-async function pollTask(token: string, taskId: string): Promise<string> {+function waitForPollInterval(signal?: AbortSignal): Promise<void> {+ throwIfAborted(signal)+ return new Promise((resolve, reject) => {+ const timer = setTimeout(() => {+ signal?.removeEventListener("abort", onAbort)+ resolve()+ }, POLL_INTERVAL_MS)+ const onAbort = () => {+ clearTimeout(timer)+ signal?.removeEventListener("abort", onAbort)+ reject(new Error("MinerU parsing cancelled"))+ }+ signal?.addEventListener("abort", onAbort, { once: true })+ })+}++async function pollTask(token: string, taskId: string, signal?: AbortSignal): Promise<string> {const httpFetch = await getHttpFetch()const headers = await mineruHeaders(token)const start = Date.now()while (Date.now() - start < POLL_TIMEOUT_MS) {+ throwIfAborted(signal)const res = await httpFetch(`${API_BASE}/extract/task/${taskId}`, {headers,+ signal,})if (!res.ok) throw new Error(`MinerU poll failed: HTTP ${res.status}`)const json: TaskStatus = await res.json()+ assertMineruSuccess(json)if (json.data.state === "done" && json.data.full_zip_url) {return json.data.full_zip_url@@ -138,7 +215,7 @@ async function pollTask(token: string, taskId: string): Promise<string> {throw new Error(`MinerU parsing failed: ${json.data.err_msg ?? "unknown error"}`)}- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))+ await waitForPollInterval(signal)}throw new Error("MinerU parsing timed out after 5 minutes")@@ -147,18 +224,21 @@ async function pollTask(token: string, taskId: string): Promise<string> {async function pollBatchTask(token: string,batchId: string,+ signal?: AbortSignal,): Promise<string> {const httpFetch = await getHttpFetch()const headers = await mineruHeaders(token)const start = Date.now()while (Date.now() - start < POLL_TIMEOUT_MS) {+ throwIfAborted(signal)const res = await httpFetch(`${API_BASE}/extract-results/batch/${batchId}`,- { headers },+ { headers, signal },)if (!res.ok) throw new Error(`MinerU batch poll failed: HTTP ${res.status}`)const json: BatchStatus = await res.json()+ assertMineruSuccess(json)const result = json.data.extract_result[0]if (result?.state === "done" && result.full_zip_url) {@@ -168,21 +248,24 @@ async function pollBatchTask(throw new Error(`MinerU parsing failed: ${result.err_msg ?? "unknown error"}`)}- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))+ await waitForPollInterval(signal)}throw new Error("MinerU parsing timed out after 5 minutes")}-async function downloadAndExtractMarkdown(zipUrl: string): Promise<string> {+async function downloadAndExtractMarkdown(zipUrl: string, signal?: AbortSignal): Promise<string> {const httpFetch = await getHttpFetch()- const res = await httpFetch(zipUrl)+ throwIfAborted(signal)+ const res = await httpFetch(zipUrl, { signal })if (!res.ok) throw new Error(`MinerU zip download failed: HTTP ${res.status}`)const buffer = await res.arrayBuffer()const zip = await JSZip.loadAsync(buffer)- // Look for .md file in the zip (MinerU typically outputs full.md or <name>.md)+ // Official MinerU result archives contain full.md. Prefer it; fall+ // back to another Markdown file only for compatibility with older or+ // unusual archives.const mdEntries: [string, JSZip.JSZipObject][] = []zip.forEach((relativePath, file) => {if (!file.dir && relativePath.endsWith(".md")) {@@ -194,7 +277,10 @@ async function downloadAndExtractMarkdown(zipUrl: string): Promise<string> {throw new Error("No Markdown file found in MinerU result zip")}- return await mdEntries[0][1].async("string")+ const fullMd = mdEntries.find(([relativePath]) =>+ relativePath.split("/").pop()?.toLowerCase() === "full.md"+ )+ return await (fullMd ?? mdEntries[0])[1].async("string")}// ── Public API ──@@ -213,21 +299,32 @@ export async function parseWithMineru(sourcePath: string,sourceUrl?: string,onProgress?: (msg: string) => void,+ signal?: AbortSignal,): Promise<string> {+ throwIfAborted(signal)if (!config.token) throw new Error("MinerU API token not configured")+ if (config.modelVersion !== "pipeline" && config.modelVersion !== "vlm") {+ throw new Error("MinerU PDF parsing supports only pipeline or vlm model versions")+ }let zipUrl: stringif (sourceUrl) {onProgress?.("Submitting URL to MinerU...")- const taskId = await submitUrlTask(config.token, sourceUrl, config.modelVersion)+ const taskId = await submitUrlTask(config.token, sourceUrl, config.modelVersion, signal)onProgress?.("Waiting for MinerU to finish...")- zipUrl = await pollTask(config.token, taskId)+ zipUrl = await pollTask(config.token, taskId, signal)} else {onProgress?.("Uploading file to MinerU...")+ throwIfAborted(signal)+ const fileSize = await getFileSize(sourcePath)+ if (fileSize > MAX_ACCURATE_PARSE_BYTES) {+ throw new Error("MinerU accurate parsing supports files up to 200 MB")+ }// Read file as base64const fileName = sourcePath.split("/").pop() ?? "document.pdf"+ throwIfAborted(signal)const { base64 } = await readFileAsBase64(sourcePath)const { batchId } = await uploadFileForTask(@@ -235,13 +332,14 @@ export async function parseWithMineru(fileName,base64,config.modelVersion,+ signal,)onProgress?.("Waiting for MinerU to finish...")- zipUrl = await pollBatchTask(config.token, batchId)+ zipUrl = await pollBatchTask(config.token, batchId, signal)}onProgress?.("Downloading parsed result...")- const markdown = await downloadAndExtractMarkdown(zipUrl)+ const markdown = await downloadAndExtractMarkdown(zipUrl, signal)onProgress?.("Done")return markdown@@ -268,7 +366,13 @@ export async function testMineruConnection(token: string): Promise<void> {}const json: TaskResponse = await res.json()- if (json.code !== 0) {- throw new Error(json.msg || "Unknown API error")- }+ assertMineruSuccess(json)+}++// Test-only hooks for MinerU's browser/Tauri boundary helpers.+export const __mineruTest = {+ downloadAndExtractMarkdown,+ mineruApiErrorMessage,+ decodeBase64ToBytes,+ MAX_ACCURATE_PARSE_BYTES,}diff --git a/src/lib/project-store.test.ts b/src/lib/project-store.test.tsnew file mode 100644index 00000000..b8a1e2c2--- /dev/null+++ b/src/lib/project-store.test.ts@@ -0,0 +1,28 @@+import { describe, expect, it } from "vitest"+import { __projectStoreTest } from "./project-store"++describe("project-store MinerU config normalization", () => {+ it("preserves valid MinerU config values", () => {+ expect(__projectStoreTest.normalizeMineruConfig({+ enabled: true,+ token: "token-123",+ modelVersion: "pipeline",+ })).toEqual({+ enabled: true,+ token: "token-123",+ modelVersion: "pipeline",+ })+ })++ it("migrates legacy and malformed MinerU config values to safe defaults", () => {+ expect(__projectStoreTest.normalizeMineruConfig({+ enabled: "yes" as unknown as boolean,+ token: 123 as unknown as string,+ modelVersion: "mineru-html" as "vlm",+ })).toEqual({+ enabled: false,+ token: "",+ modelVersion: "vlm",+ })+ })+})diff --git a/src/lib/project-store.ts b/src/lib/project-store.tsindex 85d75e58..ea342181 100644--- a/src/lib/project-store.ts+++ b/src/lib/project-store.ts@@ -112,14 +112,27 @@ export async function loadMultimodalConfig(): Promise<MultimodalConfig | null> {const MINERU_KEY = "mineruConfig"+function normalizeMineruConfig(config: MineruConfig): MineruConfig {+ return {+ enabled: config.enabled === true,+ token: typeof config.token === "string" ? config.token : "",+ modelVersion: config.modelVersion === "pipeline" ? "pipeline" : "vlm",+ }+}++export const __projectStoreTest = {+ normalizeMineruConfig,+}+export async function saveMineruConfig(config: MineruConfig): Promise<void> {const store = await getStore()- await store.set(MINERU_KEY, config)+ await store.set(MINERU_KEY, normalizeMineruConfig(config))}export async function loadMineruConfig(): Promise<MineruConfig | null> {const store = await getStore()- return (await store.get<MineruConfig>(MINERU_KEY)) ?? null+ const config = await store.get<MineruConfig>(MINERU_KEY)+ return config ? normalizeMineruConfig(config) : null}// IMPORTANT: Keep this key in sync with the Rust setup hookdiff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.tsindex 5fecbc6f..6cd18ff3 100644--- a/src/stores/wiki-store.ts+++ b/src/stores/wiki-store.ts@@ -213,7 +213,7 @@ interface SourceWatchConfig {maxFileSizeMb: number}-export type MineruModelVersion = "pipeline" | "vlm" | "mineru-html"+export type MineruModelVersion = "pipeline" | "vlm"export interface MineruConfig {enabled: boolean