Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

fix: harden MinerU PDF parsing

nash_sudf3d15fde9f4
diff --git a/src/components/settings/sections/mineru-section.tsx b/src/components/settings/sections/mineru-section.tsx
index 9ac9474a..638295a9 100644
--- a/src/components/settings/sections/mineru-section.tsx
+++ b/src/components/settings/sections/mineru-section.tsx
@@ -36,14 +36,20 @@ export function MineruSection({ draft, setDraft }: Props) {
<div className="space-y-6">
<div>
<h2 className="text-xl font-semibold">
- {t("settings.mineru", { defaultValue: "MinerU PDF Parser" })}
+ {t("settings.sections.mineru.title", { defaultValue: "MinerU PDF Parser" })}
</h2>
<p className="text-sm text-muted-foreground mt-1">
- {t("settings.mineruDescription", {
+ {t("settings.sections.mineru.description", {
defaultValue:
"Use MinerU cloud API for higher quality PDF parsing (tables, formulas, complex layouts)",
})}
</p>
+ <p className="mt-2 rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-800 dark:text-amber-200">
+ {t("settings.sections.mineru.privacyNotice", {
+ defaultValue:
+ "When enabled, PDF contents are uploaded to MinerU cloud for parsing. Do not enable this for sensitive documents unless you accept that.",
+ })}
+ </p>
</div>
<label className="flex items-center gap-2">
@@ -54,7 +60,7 @@ export function MineruSection({ draft, setDraft }: Props) {
className="h-4 w-4"
/>
<span className="text-sm">
- {t("settings.mineruEnabled", { defaultValue: "Enable MinerU" })}
+ {t("settings.sections.mineru.enabled", { defaultValue: "Enable MinerU" })}
</span>
</label>
@@ -62,7 +68,7 @@ export function MineruSection({ draft, setDraft }: Props) {
<div className="space-y-4 pl-1">
<div className="space-y-2">
<Label htmlFor="mineru-token">
- {t("settings.mineruToken", { defaultValue: "API Token" })}
+ {t("settings.sections.mineru.token", { defaultValue: "API Token" })}
</Label>
<Input
id="mineru-token"
@@ -72,7 +78,7 @@ export function MineruSection({ draft, setDraft }: Props) {
setDraft("mineruToken", e.target.value)
setTestState("idle")
}}
- placeholder={t("settings.mineruTokenHint", {
+ placeholder={t("settings.sections.mineru.tokenHint", {
defaultValue: "Get your token from mineru.net",
})}
/>
@@ -80,7 +86,7 @@ export function MineruSection({ draft, setDraft }: Props) {
<div className="space-y-2">
<Label htmlFor="mineru-model">
- {t("settings.mineruModel", { defaultValue: "Model Version" })}
+ {t("settings.sections.mineru.model", { defaultValue: "Model Version" })}
</Label>
<select
id="mineru-model"
@@ -91,22 +97,28 @@ export function MineruSection({ draft, setDraft }: Props) {
}
>
<option value="vlm">
- {t("settings.mineruModelVlm", {
+ {t("settings.sections.mineru.modelVlm", {
defaultValue: "VLM (Recommended, best for complex layouts)",
})}
</option>
<option value="pipeline">
- {t("settings.mineruModelPipeline", {
+ {t("settings.sections.mineru.modelPipeline", {
defaultValue: "Pipeline (Faster)",
})}
</option>
- <option value="mineru-html">
- {t("settings.mineruModelHtml", {
- defaultValue: "MinerU-HTML",
- })}
- </option>
</select>
+ <p className="text-xs text-muted-foreground">
+ {t("settings.sections.mineru.modelHint", {
+ defaultValue: "PDF parsing supports pipeline and vlm. MinerU-HTML is for HTML files and is not used here.",
+ })}
+ </p>
</div>
+ <p className="rounded-md bg-muted/50 px-3 py-2 text-xs text-muted-foreground">
+ {t("settings.sections.mineru.testQuotaNotice", {
+ defaultValue:
+ "Connection test submits a small demo file to MinerU and may consume a small amount of quota.",
+ })}
+ </p>
<div className="flex items-center gap-3">
<Button
@@ -118,21 +130,21 @@ export function MineruSection({ draft, setDraft }: Props) {
}
>
{testState === "running"
- ? t("settings.mineruTesting", { defaultValue: "Testing..." })
- : t("settings.mineruTest", {
+ ? t("settings.sections.mineru.testing", { defaultValue: "Testing..." })
+ : t("settings.sections.mineru.test", {
defaultValue: "Test Connection",
})}
</Button>
{testState === "success" && (
<span className="text-sm text-green-600">
- {t("settings.mineruTestSuccess", {
+ {t("settings.sections.mineru.testSuccess", {
defaultValue: "Connection successful",
})}
</span>
)}
{testState === "failed" && (
<span className="text-sm text-red-600">
- {t("settings.mineruTestFailed", {
+ {t("settings.sections.mineru.testFailed", {
defaultValue: "Test failed",
})}
: {testError}
diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx
index 4b42cbe6..921c77cd 100644
--- a/src/components/settings/settings-view.tsx
+++ b/src/components/settings/settings-view.tsx
@@ -271,6 +271,7 @@ export function SettingsView() {
proxyConfig,
scheduledImportConfig,
sourceWatchConfig,
+ mineruConfig,
apiConfig,
generalConfig,
maxHistoryMessages,
@@ -287,6 +288,7 @@ export function SettingsView() {
proxyConfig,
scheduledImportConfig,
sourceWatchConfig,
+ mineruConfig,
apiConfig,
generalConfig,
maxHistoryMessages,
diff --git a/src/i18n/en.json b/src/i18n/en.json
index 3cef23e5..1100f5a2 100644
--- a/src/i18n/en.json
+++ b/src/i18n/en.json
@@ -583,14 +583,16 @@
"mineru": {
"title": "MinerU PDF Parser",
"description": "Use MinerU cloud API for higher quality PDF parsing (tables, formulas, complex layouts). Default uses built-in pdfium extraction.",
+ "privacyNotice": "When enabled, PDF contents are uploaded to MinerU cloud for parsing. Do not enable this for sensitive documents unless you accept that.",
"enabled": "Enable MinerU",
"token": "API Token",
"tokenHint": "Get your token from mineru.net",
"model": "Model Version",
"modelVlm": "VLM (Recommended, best for complex layouts)",
"modelPipeline": "Pipeline (Faster)",
- "modelHtml": "MinerU-HTML",
+ "modelHint": "PDF parsing supports pipeline and vlm. MinerU-HTML is for HTML files and is not used here.",
"test": "Test Connection",
+ "testQuotaNotice": "Connection test submits a small demo file to MinerU and may consume a small amount of quota.",
"testing": "Testing...",
"testSuccess": "Connection successful",
"testFailed": "Test failed"
diff --git a/src/i18n/zh.json b/src/i18n/zh.json
index 7a0eb001..c61862da 100644
--- a/src/i18n/zh.json
+++ b/src/i18n/zh.json
@@ -583,14 +583,16 @@
"mineru": {
"title": "MinerU PDF 解析",
"description": "使用 MinerU 云端 API 获得更高质量的 PDF 解析(表格、公式、复杂排版)。默认使用内置 pdfium 提取。",
+ "privacyNotice": "启用后,PDF 内容会上传到 MinerU 云端进行解析。除非你接受这一点,否则不要用于敏感文档。",
"enabled": "启用 MinerU",
"token": "API Token",
"tokenHint": "从 mineru.net 获取您的 Token",
"model": "模型版本",
"modelVlm": "VLM(推荐,适合复杂排版)",
"modelPipeline": "Pipeline(更快)",
- "modelHtml": "MinerU-HTML",
+ "modelHint": "PDF 解析仅使用 pipeline 或 vlm。MinerU-HTML 用于 HTML 文件,这里不会使用。",
"test": "测试连接",
+ "testQuotaNotice": "连接测试会向 MinerU 提交一个小型示例文件,可能消耗少量额度。",
"testing": "测试中...",
"testSuccess": "连接成功",
"testFailed": "测试失败"
diff --git a/src/lib/ingest-source-path-collision.test.ts b/src/lib/ingest-source-path-collision.test.ts
index 99d5115c..b293d9ab 100644
--- a/src/lib/ingest-source-path-collision.test.ts
+++ b/src/lib/ingest-source-path-collision.test.ts
@@ -108,10 +108,16 @@ vi.mock("./llm-client", () => ({
}),
}))
+vi.mock("./mineru", () => ({
+ parseWithMineru: vi.fn(),
+}))
+
import { autoIngest, executeIngestWrites } from "./ingest"
import { streamChat } from "./llm-client"
+import { parseWithMineru } from "./mineru"
const mockStreamChat = vi.mocked(streamChat)
+const mockParseWithMineru = vi.mocked(parseWithMineru)
describe("autoIngest source summary paths", () => {
let tmp: { path: string; cleanup: () => Promise<void> } | undefined
@@ -123,6 +129,7 @@ describe("autoIngest source summary paths", () => {
generationSuffix = ""
abortDuringReview = null
mockStreamChat.mockClear()
+ mockParseWithMineru.mockReset()
tmp = await createTempProject("same-basename-sources")
await writeFileRaw(`${tmp.path}/purpose.md`, "# Purpose\n\nTrack project config files.\n")
@@ -483,6 +490,69 @@ describe("autoIngest source summary paths", () => {
).rejects.toThrow("AbortError")
})
+ it("falls back to built-in PDF extraction when MinerU fails for a non-cancelled ingest", async () => {
+ if (!tmp) throw new Error("missing temp project")
+ sourceMarkers = ["mineru fallback source"]
+ await writeFileRaw(`${tmp.path}/raw/sources/project-a/report.pdf`, "pdf fallback text\n")
+ useWikiStore.setState({
+ mineruConfig: {
+ enabled: true,
+ token: "mineru-token",
+ modelVersion: "vlm",
+ },
+ })
+ mockParseWithMineru.mockRejectedValueOnce(new Error("network failure from MinerU"))
+ const updateSpy = vi.spyOn(useActivityStore.getState(), "updateItem")
+
+ const written = await autoIngest(
+ tmp.path,
+ `${tmp.path}/raw/sources/project-a/report.pdf`,
+ useWikiStore.getState().llmConfig,
+ undefined,
+ "project-a",
+ )
+
+ expect(written.length).toBeGreaterThan(0)
+ expect(mockParseWithMineru).toHaveBeenCalled()
+ expect(
+ updateSpy.mock.calls.some(([, updates]) =>
+ updates.detail?.includes("falling back to built-in PDF extraction"),
+ ),
+ ).toBe(true)
+ updateSpy.mockRestore()
+ })
+
+ it("does not fall back to built-in PDF extraction when MinerU is cancelled", async () => {
+ if (!tmp) throw new Error("missing temp project")
+ await writeFileRaw(`${tmp.path}/raw/sources/project-a/cancelled.pdf`, "pdf fallback text\n")
+ useWikiStore.setState({
+ mineruConfig: {
+ enabled: true,
+ token: "mineru-token",
+ modelVersion: "vlm",
+ },
+ })
+ const controller = new AbortController()
+ controller.abort()
+ mockParseWithMineru.mockRejectedValueOnce(new Error("MinerU parsing cancelled"))
+
+ await expect(
+ autoIngest(
+ tmp.path,
+ `${tmp.path}/raw/sources/project-a/cancelled.pdf`,
+ useWikiStore.getState().llmConfig,
+ controller.signal,
+ "project-a",
+ ),
+ ).rejects.toThrow("MinerU parsing cancelled")
+
+ expect(
+ useActivityStore.getState().items.some((item) =>
+ item.detail?.includes("falling back to built-in PDF extraction"),
+ ),
+ ).toBe(false)
+ })
+
it("canonicalizes interactive source summary paths and sources frontmatter", async () => {
if (!tmp) throw new Error("missing temp project")
diff --git a/src/lib/ingest.ts b/src/lib/ingest.ts
index c4ec78ab..8393ce84 100644
--- a/src/lib/ingest.ts
+++ b/src/lib/ingest.ts
@@ -483,6 +483,7 @@ async function autoIngestImpl(
const isPdf = lowerExt === "pdf"
const mineruCfg = useWikiStore.getState().mineruConfig
if (isPdf && mineruCfg.enabled && mineruCfg.token) {
+ let mineruSucceeded = false
try {
const cacheDir = sp.substring(0, sp.lastIndexOf("/"))
const cachePath = `${cacheDir}/.cache/${fileName}.txt`
@@ -490,15 +491,22 @@ async function autoIngestImpl(
console.log(`[ingest:mineru] submitting "${fileName}" to MinerU API`)
const markdown = await parseWithMineru(mineruCfg, sp, undefined, (msg) => {
activity.updateItem(activityId, { detail: `MinerU: ${msg}` })
- })
+ }, signal)
await createDirectory(`${cacheDir}/.cache`)
await writeFile(cachePath, markdown)
+ mineruSucceeded = true
console.log(`[ingest:mineru] cached MinerU output for "${fileName}" (${markdown.length} chars)`)
} catch (err) {
- const msg = err instanceof Error ? err.message : String(err)
+ if (signal?.aborted) throw err
+ const msg = trimInlineStatus(err instanceof Error ? err.message : String(err))
console.warn(`[ingest:mineru] MinerU parsing failed, falling back to pdfium: ${msg}`)
+ activity.updateItem(activityId, {
+ detail: `MinerU failed, falling back to built-in PDF extraction: ${msg}`,
+ })
+ }
+ if (mineruSucceeded && !signal?.aborted) {
+ activity.updateItem(activityId, { detail: "Reading source..." })
}
- activity.updateItem(activityId, { detail: "Reading source..." })
}
const [sourceContent, schema, purpose, index, overview] = await Promise.all([
@@ -1976,6 +1984,10 @@ function trimLongText(text: string, maxChars: number): string {
return `${text.slice(0, maxChars).trimEnd()}\n\n[...trimmed for prompt budget...]`
}
+function trimInlineStatus(text: string, maxChars = 240): string {
+ return text.length <= maxChars ? text : `${text.slice(0, maxChars).trimEnd()}...`
+}
+
function hashTextHex(text: string): string {
// 64-bit FNV-1a over UTF-16 code units. This is a stability key, not
// a security primitive; validation also checks source length/chunk
diff --git a/src/lib/mineru.test.ts b/src/lib/mineru.test.ts
new file mode 100644
index 00000000..e3f4a73f
--- /dev/null
+++ b/src/lib/mineru.test.ts
@@ -0,0 +1,333 @@
+import JSZip from "jszip"
+import { beforeEach, describe, expect, it, vi } from "vitest"
+
+const mockHttpFetch = vi.fn<(url: string, opts?: RequestInit) => Promise<Response>>()
+
+vi.mock("@/lib/tauri-fetch", () => ({
+ getHttpFetch: () => Promise.resolve(mockHttpFetch),
+}))
+
+const fsMocks = vi.hoisted(() => ({
+ getFileSize: vi.fn<() => Promise<number>>(),
+ readFileAsBase64: vi.fn<() => Promise<{ base64: string; mimeType: string }>>(),
+}))
+
+vi.mock("@/commands/fs", () => ({
+ getFileSize: fsMocks.getFileSize,
+ readFileAsBase64: fsMocks.readFileAsBase64,
+}))
+
+import { __mineruTest, parseWithMineru, testMineruConnection } from "./mineru"
+
+function jsonResponse(body: unknown, init?: ResponseInit): Response {
+ return new Response(JSON.stringify(body), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ ...init,
+ })
+}
+
+async function zipResponse(files: Record<string, string>): Promise<Response> {
+ const zip = new JSZip()
+ for (const [path, content] of Object.entries(files)) {
+ zip.file(path, content)
+ }
+ const bytes = await zip.generateAsync({ type: "uint8array" })
+ const buffer = bytes.buffer.slice(
+ bytes.byteOffset,
+ bytes.byteOffset + bytes.byteLength,
+ ) as ArrayBuffer
+ return new Response(buffer)
+}
+
+beforeEach(() => {
+ mockHttpFetch.mockReset()
+ fsMocks.getFileSize.mockReset()
+ fsMocks.readFileAsBase64.mockReset()
+ fsMocks.getFileSize.mockResolvedValue(1024)
+ fsMocks.readFileAsBase64.mockResolvedValue({
+ base64: btoa("pdf bytes"),
+ mimeType: "application/pdf",
+ })
+})
+
+describe("MinerU API helpers", () => {
+ it("maps official API error codes to actionable messages", () => {
+ expect(__mineruTest.mineruApiErrorMessage("A0202", "bad token")).toContain("invalid")
+ expect(__mineruTest.mineruApiErrorMessage("A0211", "expired")).toContain("expired")
+ expect(__mineruTest.mineruApiErrorMessage(-60005, "too large")).toContain("200 MB")
+ expect(__mineruTest.mineruApiErrorMessage(-60006, "too many pages")).toContain("200 page")
+ expect(__mineruTest.mineruApiErrorMessage(-60018, "quota")).toContain("quota")
+ expect(__mineruTest.mineruApiErrorMessage(123, "other")).toBe("MinerU API error 123: other")
+ })
+
+ it("prefers full.md from MinerU result zip", async () => {
+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({
+ "result/other.md": "other markdown",
+ "result/full.md": "full markdown",
+ }))
+
+ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))
+ .resolves.toBe("full markdown")
+ })
+
+ it("falls back to another markdown file when full.md is missing", async () => {
+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({
+ "result/page.md": "fallback markdown",
+ }))
+
+ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))
+ .resolves.toBe("fallback markdown")
+ })
+
+ it("rejects MinerU zip files without markdown output", async () => {
+ mockHttpFetch.mockResolvedValueOnce(await zipResponse({
+ "result/layout.json": "{}",
+ }))
+
+ await expect(__mineruTest.downloadAndExtractMarkdown("https://cdn/result.zip"))
+ .rejects.toThrow("No Markdown file")
+ })
+})
+
+describe("parseWithMineru", () => {
+ it("rejects unsupported MinerU model versions before reading or uploading", async () => {
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "mineru-html" as "vlm",
+ }, "/tmp/doc.pdf")).rejects.toThrow("pipeline or vlm")
+
+ expect(fsMocks.getFileSize).not.toHaveBeenCalled()
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ })
+
+ it("rejects local files over MinerU's 200 MB accurate parsing limit before upload", async () => {
+ fsMocks.getFileSize.mockResolvedValue(__mineruTest.MAX_ACCURATE_PARSE_BYTES + 1)
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/large.pdf")).rejects.toThrow("200 MB")
+
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ expect(fsMocks.readFileAsBase64).not.toHaveBeenCalled()
+ })
+
+ it("rejects before network access when the abort signal is already cancelled", async () => {
+ const controller = new AbortController()
+ controller.abort()
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf", undefined, undefined, controller.signal)).rejects.toThrow("cancelled")
+
+ expect(mockHttpFetch).not.toHaveBeenCalled()
+ })
+
+ it("rejects batch upload responses without an upload URL", async () => {
+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", file_urls: [] },
+ }))
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf")).rejects.toThrow("upload URL")
+ })
+
+ it("uploads the decoded PDF bytes to the MinerU upload URL", async () => {
+ fsMocks.readFileAsBase64.mockResolvedValueOnce({
+ base64: btoa("custom pdf bytes"),
+ mimeType: "application/pdf",
+ })
+ mockHttpFetch
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },
+ }))
+ .mockResolvedValueOnce(new Response("", { status: 200 }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: {
+ batch_id: "batch-1",
+ extract_result: [{ file_name: "doc.pdf", state: "done", full_zip_url: "https://zip" }],
+ },
+ }))
+ .mockResolvedValueOnce(await zipResponse({ "full.md": "parsed markdown" }))
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf")).resolves.toBe("parsed markdown")
+
+ const uploadBody = mockHttpFetch.mock.calls[1]?.[1]?.body
+ expect(uploadBody).toBeInstanceOf(ArrayBuffer)
+ expect(new TextDecoder().decode(uploadBody as ArrayBuffer)).toBe("custom pdf bytes")
+ })
+
+ it("submits URL tasks without reading or uploading a local file", async () => {
+ mockHttpFetch
+ .mockResolvedValueOnce(jsonResponse({
+ code: "0",
+ msg: "ok",
+ data: { task_id: "task-1" },
+ }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { task_id: "task-1", state: "done", full_zip_url: "https://zip" },
+ }))
+ .mockResolvedValueOnce(await zipResponse({ "full.md": "url markdown" }))
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "pipeline",
+ }, "/tmp/doc.pdf", "https://example.test/doc.pdf")).resolves.toBe("url markdown")
+
+ expect(fsMocks.getFileSize).not.toHaveBeenCalled()
+ expect(fsMocks.readFileAsBase64).not.toHaveBeenCalled()
+ expect(JSON.parse(String(mockHttpFetch.mock.calls[0]?.[1]?.body))).toMatchObject({
+ url: "https://example.test/doc.pdf",
+ model_version: "pipeline",
+ })
+ })
+
+ it("rejects MinerU failed states with the service error message", async () => {
+ mockHttpFetch
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },
+ }))
+ .mockResolvedValueOnce(new Response("", { status: 200 }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: {
+ batch_id: "batch-1",
+ extract_result: [{ file_name: "doc.pdf", state: "failed", err_msg: "parse exploded" }],
+ },
+ }))
+
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf")).rejects.toThrow("parse exploded")
+ })
+
+ it("stops polling immediately when the abort signal fires during the poll interval", async () => {
+ mockHttpFetch
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },
+ }))
+ .mockResolvedValueOnce(new Response("", { status: 200 }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "running" }] },
+ }))
+
+ const controller = new AbortController()
+ const result = parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf", undefined, undefined, controller.signal)
+
+ setTimeout(() => controller.abort(), 10)
+
+ await expect(result).rejects.toThrow("cancelled")
+ expect(mockHttpFetch).toHaveBeenCalledTimes(3)
+ })
+
+ it("handles official pending, waiting-file, converting, and running states before completion", async () => {
+ mockHttpFetch
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", file_urls: ["https://upload"] },
+ }))
+ .mockResolvedValueOnce(new Response("", { status: 200 }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "pending" }] },
+ }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "waiting-file" }] },
+ }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "converting" }] },
+ }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: { batch_id: "batch-1", extract_result: [{ file_name: "doc.pdf", state: "running" }] },
+ }))
+ .mockResolvedValueOnce(jsonResponse({
+ code: 0,
+ msg: "ok",
+ data: {
+ batch_id: "batch-1",
+ extract_result: [{ file_name: "doc.pdf", state: "done", full_zip_url: "https://zip" }],
+ },
+ }))
+ .mockResolvedValueOnce(await zipResponse({ "full.md": "parsed markdown" }))
+
+ const progress: string[] = []
+ await expect(parseWithMineru({
+ enabled: true,
+ token: "token",
+ modelVersion: "vlm",
+ }, "/tmp/doc.pdf", undefined, (msg) => progress.push(msg))).resolves.toBe("parsed markdown")
+
+ expect(progress).toContain("Waiting for MinerU to finish...")
+ expect(mockHttpFetch).toHaveBeenCalledTimes(8)
+ }, 16_000)
+})
+
+describe("testMineruConnection", () => {
+ it("resolves when MinerU accepts the connection test task", async () => {
+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({
+ code: "0",
+ msg: "ok",
+ data: { task_id: "task-1" },
+ }))
+
+ await expect(testMineruConnection("token")).resolves.toBeUndefined()
+ })
+
+ it("includes HTTP status and response body when connection test transport fails", async () => {
+ mockHttpFetch.mockResolvedValueOnce(new Response("bad gateway", { status: 502 }))
+
+ await expect(testMineruConnection("token")).rejects.toThrow("HTTP 502: bad gateway")
+ })
+
+ it("maps MinerU API errors during connection test", async () => {
+ mockHttpFetch.mockResolvedValueOnce(jsonResponse({
+ code: "A0202",
+ msg: "token invalid",
+ data: {},
+ }))
+
+ await expect(testMineruConnection("bad-token")).rejects.toThrow("invalid")
+ })
+})
diff --git a/src/lib/mineru.ts b/src/lib/mineru.ts
index 75c02e1e..f72f9002 100644
--- a/src/lib/mineru.ts
+++ b/src/lib/mineru.ts
@@ -1,25 +1,28 @@
import JSZip from "jszip"
import type { MineruConfig } from "@/stores/wiki-store"
-import { readFileAsBase64 } from "@/commands/fs"
+import { getFileSize, readFileAsBase64 } from "@/commands/fs"
import { getHttpFetch } from "@/lib/tauri-fetch"
const API_BASE = "https://mineru.net/api/v4"
const POLL_INTERVAL_MS = 3_000
const POLL_TIMEOUT_MS = 300_000 // 5 minutes
+const MAX_ACCURATE_PARSE_BYTES = 200 * 1024 * 1024
// ── Types ──
interface TaskResponse {
- code: number
+ code: number | string
data: { task_id: string }
msg: string
}
+type MineruTaskState = "pending" | "running" | "converting" | "done" | "failed" | "waiting-file"
+
interface TaskStatus {
- code: number
+ code: number | string
data: {
task_id: string
- state: "running" | "done" | "failed"
+ state: MineruTaskState
full_zip_url?: string
err_msg?: string
extract_progress?: { extracted_pages: number; total_pages: number }
@@ -28,12 +31,12 @@ interface TaskStatus {
}
interface BatchStatus {
- code: number
+ code: number | string
data: {
batch_id: string
extract_result: Array<{
file_name: string
- state: "running" | "done" | "failed"
+ state: MineruTaskState
full_zip_url?: string
err_msg?: string
}>
@@ -42,7 +45,7 @@ interface BatchStatus {
}
interface UploadUrlResponse {
- code: number
+ code: number | string
data: {
batch_id: string
file_urls: string[]
@@ -59,20 +62,71 @@ async function mineruHeaders(token: string): Promise<HeadersInit> {
}
}
+function mineruApiErrorMessage(code: number | string | undefined, msg?: string): string {
+ const key = String(code ?? "")
+ const known: Record<string, string> = {
+ A0202: "MinerU token is invalid. Check the API token in Settings.",
+ A0211: "MinerU token has expired. Create a new API token and update Settings.",
+ "-60005": "MinerU rejected the file because it is larger than 200 MB.",
+ "-60006": "MinerU rejected the file because it exceeds the 200 page limit.",
+ "-60018": "MinerU daily parsing quota has been reached.",
+ }
+ const knownMessage = known[key]
+ if (knownMessage) return msg ? `${knownMessage} (${msg})` : knownMessage
+ return msg ? `MinerU API error ${key || "unknown"}: ${msg}` : `MinerU API error ${key || "unknown"}`
+}
+
+function assertMineruSuccess(json: { code: number | string; msg?: string }): void {
+ if (json.code !== 0 && json.code !== "0") {
+ throw new Error(mineruApiErrorMessage(json.code, json.msg))
+ }
+}
+
+function throwIfAborted(signal?: AbortSignal): void {
+ if (signal?.aborted) {
+ throw new Error("MinerU parsing cancelled")
+ }
+}
+
+function decodeBase64ToBytes(base64: string): Uint8Array {
+ if (typeof atob !== "function") {
+ throw new Error("Base64 decoding is not available in this runtime")
+ }
+ const binaryStr = atob(base64)
+ const bytes = new Uint8Array(binaryStr.length)
+ for (let i = 0; i < binaryStr.length; i++) {
+ bytes[i] = binaryStr.charCodeAt(i)
+ }
+ return bytes
+}
+
+function bytesToUploadBody(bytes: Uint8Array): ArrayBuffer {
+ if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
+ return bytes.buffer as ArrayBuffer
+ }
+ return bytes.buffer.slice(
+ bytes.byteOffset,
+ bytes.byteOffset + bytes.byteLength,
+ ) as ArrayBuffer
+}
+
async function submitUrlTask(
token: string,
url: string,
modelVersion: string,
+ signal?: AbortSignal,
): Promise<string> {
const httpFetch = await getHttpFetch()
+ throwIfAborted(signal)
const res = await httpFetch(`${API_BASE}/extract/task`, {
method: "POST",
headers: await mineruHeaders(token),
+ signal,
body: JSON.stringify({ url, model_version: modelVersion }),
})
if (!res.ok) throw new Error(`MinerU submit failed: HTTP ${res.status}`)
const json: TaskResponse = await res.json()
- if (json.code !== 0) throw new Error(`MinerU API error: ${json.msg}`)
+ assertMineruSuccess(json)
return json.data.task_id
}
@@ -81,14 +135,17 @@ async function uploadFileForTask(
fileName: string,
fileBase64: string,
modelVersion: string,
+ signal?: AbortSignal,
): Promise<{ batchId: string; uploadUrl: string }> {
const httpFetch = await getHttpFetch()
const headers = await mineruHeaders(token)
+ throwIfAborted(signal)
// Step 1: Get upload URL
const res = await httpFetch(`${API_BASE}/file-urls/batch`, {
method: "POST",
headers,
+ signal,
body: JSON.stringify({
files: [{ name: fileName, data_id: fileName }],
model_version: modelVersion,
@@ -96,21 +153,22 @@ async function uploadFileForTask(
})
if (!res.ok) throw new Error(`MinerU batch submit failed: HTTP ${res.status}`)
const json: UploadUrlResponse = await res.json()
- if (json.code !== 0) throw new Error(`MinerU API error: ${json.msg}`)
+ assertMineruSuccess(json)
const batchId = json.data.batch_id
const uploadUrl = json.data.file_urls[0]
+ if (!batchId || !uploadUrl) {
+ throw new Error("MinerU did not return a file upload URL")
+ }
// Step 2: Upload file binary (convert base64 back to binary)
- const binaryStr = atob(fileBase64)
- const bytes = new Uint8Array(binaryStr.length)
- for (let i = 0; i < binaryStr.length; i++) {
- bytes[i] = binaryStr.charCodeAt(i)
- }
+ const bytes = decodeBase64ToBytes(fileBase64)
+ throwIfAborted(signal)
const uploadRes = await httpFetch(uploadUrl, {
method: "PUT",
- body: bytes,
+ signal,
+ body: bytesToUploadBody(bytes),
})
if (!uploadRes.ok && uploadRes.status !== 200 && uploadRes.status !== 201) {
throw new Error(`MinerU file upload failed: HTTP ${uploadRes.status}`)
@@ -119,17 +177,36 @@ async function uploadFileForTask(
return { batchId, uploadUrl }
}
-async function pollTask(token: string, taskId: string): Promise<string> {
+function waitForPollInterval(signal?: AbortSignal): Promise<void> {
+ throwIfAborted(signal)
+ return new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ signal?.removeEventListener("abort", onAbort)
+ resolve()
+ }, POLL_INTERVAL_MS)
+ const onAbort = () => {
+ clearTimeout(timer)
+ signal?.removeEventListener("abort", onAbort)
+ reject(new Error("MinerU parsing cancelled"))
+ }
+ signal?.addEventListener("abort", onAbort, { once: true })
+ })
+}
+
+async function pollTask(token: string, taskId: string, signal?: AbortSignal): Promise<string> {
const httpFetch = await getHttpFetch()
const headers = await mineruHeaders(token)
const start = Date.now()
while (Date.now() - start < POLL_TIMEOUT_MS) {
+ throwIfAborted(signal)
const res = await httpFetch(`${API_BASE}/extract/task/${taskId}`, {
headers,
+ signal,
})
if (!res.ok) throw new Error(`MinerU poll failed: HTTP ${res.status}`)
const json: TaskStatus = await res.json()
+ assertMineruSuccess(json)
if (json.data.state === "done" && json.data.full_zip_url) {
return json.data.full_zip_url
@@ -138,7 +215,7 @@ async function pollTask(token: string, taskId: string): Promise<string> {
throw new Error(`MinerU parsing failed: ${json.data.err_msg ?? "unknown error"}`)
}
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
+ await waitForPollInterval(signal)
}
throw new Error("MinerU parsing timed out after 5 minutes")
@@ -147,18 +224,21 @@ async function pollTask(token: string, taskId: string): Promise<string> {
async function pollBatchTask(
token: string,
batchId: string,
+ signal?: AbortSignal,
): Promise<string> {
const httpFetch = await getHttpFetch()
const headers = await mineruHeaders(token)
const start = Date.now()
while (Date.now() - start < POLL_TIMEOUT_MS) {
+ throwIfAborted(signal)
const res = await httpFetch(
`${API_BASE}/extract-results/batch/${batchId}`,
- { headers },
+ { headers, signal },
)
if (!res.ok) throw new Error(`MinerU batch poll failed: HTTP ${res.status}`)
const json: BatchStatus = await res.json()
+ assertMineruSuccess(json)
const result = json.data.extract_result[0]
if (result?.state === "done" && result.full_zip_url) {
@@ -168,21 +248,24 @@ async function pollBatchTask(
throw new Error(`MinerU parsing failed: ${result.err_msg ?? "unknown error"}`)
}
- await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
+ await waitForPollInterval(signal)
}
throw new Error("MinerU parsing timed out after 5 minutes")
}
-async function downloadAndExtractMarkdown(zipUrl: string): Promise<string> {
+async function downloadAndExtractMarkdown(zipUrl: string, signal?: AbortSignal): Promise<string> {
const httpFetch = await getHttpFetch()
- const res = await httpFetch(zipUrl)
+ throwIfAborted(signal)
+ const res = await httpFetch(zipUrl, { signal })
if (!res.ok) throw new Error(`MinerU zip download failed: HTTP ${res.status}`)
const buffer = await res.arrayBuffer()
const zip = await JSZip.loadAsync(buffer)
- // Look for .md file in the zip (MinerU typically outputs full.md or <name>.md)
+ // Official MinerU result archives contain full.md. Prefer it; fall
+ // back to another Markdown file only for compatibility with older or
+ // unusual archives.
const mdEntries: [string, JSZip.JSZipObject][] = []
zip.forEach((relativePath, file) => {
if (!file.dir && relativePath.endsWith(".md")) {
@@ -194,7 +277,10 @@ async function downloadAndExtractMarkdown(zipUrl: string): Promise<string> {
throw new Error("No Markdown file found in MinerU result zip")
}
- return await mdEntries[0][1].async("string")
+ const fullMd = mdEntries.find(([relativePath]) =>
+ relativePath.split("/").pop()?.toLowerCase() === "full.md"
+ )
+ return await (fullMd ?? mdEntries[0])[1].async("string")
}
// ── Public API ──
@@ -213,21 +299,32 @@ export async function parseWithMineru(
sourcePath: string,
sourceUrl?: string,
onProgress?: (msg: string) => void,
+ signal?: AbortSignal,
): Promise<string> {
+ throwIfAborted(signal)
if (!config.token) throw new Error("MinerU API token not configured")
+ if (config.modelVersion !== "pipeline" && config.modelVersion !== "vlm") {
+ throw new Error("MinerU PDF parsing supports only pipeline or vlm model versions")
+ }
let zipUrl: string
if (sourceUrl) {
onProgress?.("Submitting URL to MinerU...")
- const taskId = await submitUrlTask(config.token, sourceUrl, config.modelVersion)
+ const taskId = await submitUrlTask(config.token, sourceUrl, config.modelVersion, signal)
onProgress?.("Waiting for MinerU to finish...")
- zipUrl = await pollTask(config.token, taskId)
+ zipUrl = await pollTask(config.token, taskId, signal)
} else {
onProgress?.("Uploading file to MinerU...")
+ throwIfAborted(signal)
+ const fileSize = await getFileSize(sourcePath)
+ if (fileSize > MAX_ACCURATE_PARSE_BYTES) {
+ throw new Error("MinerU accurate parsing supports files up to 200 MB")
+ }
// Read file as base64
const fileName = sourcePath.split("/").pop() ?? "document.pdf"
+ throwIfAborted(signal)
const { base64 } = await readFileAsBase64(sourcePath)
const { batchId } = await uploadFileForTask(
@@ -235,13 +332,14 @@ export async function parseWithMineru(
fileName,
base64,
config.modelVersion,
+ signal,
)
onProgress?.("Waiting for MinerU to finish...")
- zipUrl = await pollBatchTask(config.token, batchId)
+ zipUrl = await pollBatchTask(config.token, batchId, signal)
}
onProgress?.("Downloading parsed result...")
- const markdown = await downloadAndExtractMarkdown(zipUrl)
+ const markdown = await downloadAndExtractMarkdown(zipUrl, signal)
onProgress?.("Done")
return markdown
@@ -268,7 +366,13 @@ export async function testMineruConnection(token: string): Promise<void> {
}
const json: TaskResponse = await res.json()
- if (json.code !== 0) {
- throw new Error(json.msg || "Unknown API error")
- }
+ assertMineruSuccess(json)
+}
+
+// Test-only hooks for MinerU's browser/Tauri boundary helpers.
+export const __mineruTest = {
+ downloadAndExtractMarkdown,
+ mineruApiErrorMessage,
+ decodeBase64ToBytes,
+ MAX_ACCURATE_PARSE_BYTES,
}
diff --git a/src/lib/project-store.test.ts b/src/lib/project-store.test.ts
new file mode 100644
index 00000000..b8a1e2c2
--- /dev/null
+++ b/src/lib/project-store.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest"
+import { __projectStoreTest } from "./project-store"
+
+describe("project-store MinerU config normalization", () => {
+ it("preserves valid MinerU config values", () => {
+ expect(__projectStoreTest.normalizeMineruConfig({
+ enabled: true,
+ token: "token-123",
+ modelVersion: "pipeline",
+ })).toEqual({
+ enabled: true,
+ token: "token-123",
+ modelVersion: "pipeline",
+ })
+ })
+
+ it("migrates legacy and malformed MinerU config values to safe defaults", () => {
+ expect(__projectStoreTest.normalizeMineruConfig({
+ enabled: "yes" as unknown as boolean,
+ token: 123 as unknown as string,
+ modelVersion: "mineru-html" as "vlm",
+ })).toEqual({
+ enabled: false,
+ token: "",
+ modelVersion: "vlm",
+ })
+ })
+})
diff --git a/src/lib/project-store.ts b/src/lib/project-store.ts
index 85d75e58..ea342181 100644
--- a/src/lib/project-store.ts
+++ b/src/lib/project-store.ts
@@ -112,14 +112,27 @@ export async function loadMultimodalConfig(): Promise<MultimodalConfig | null> {
const MINERU_KEY = "mineruConfig"
+function normalizeMineruConfig(config: MineruConfig): MineruConfig {
+ return {
+ enabled: config.enabled === true,
+ token: typeof config.token === "string" ? config.token : "",
+ modelVersion: config.modelVersion === "pipeline" ? "pipeline" : "vlm",
+ }
+}
+
+export const __projectStoreTest = {
+ normalizeMineruConfig,
+}
+
export async function saveMineruConfig(config: MineruConfig): Promise<void> {
const store = await getStore()
- await store.set(MINERU_KEY, config)
+ await store.set(MINERU_KEY, normalizeMineruConfig(config))
}
export async function loadMineruConfig(): Promise<MineruConfig | null> {
const store = await getStore()
- return (await store.get<MineruConfig>(MINERU_KEY)) ?? null
+ const config = await store.get<MineruConfig>(MINERU_KEY)
+ return config ? normalizeMineruConfig(config) : null
}
// IMPORTANT: Keep this key in sync with the Rust setup hook
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index 5fecbc6f..6cd18ff3 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -213,7 +213,7 @@ interface SourceWatchConfig {
maxFileSizeMb: number
}
-export type MineruModelVersion = "pipeline" | "vlm" | "mineru-html"
+export type MineruModelVersion = "pipeline" | "vlm"
export interface MineruConfig {
enabled: boolean