Skip to content

LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。

TypeScript

feat: support self-hosted local MinerU backend for PDF parsing

polard2aa0856dcda5
diff --git a/src/components/settings/sections/mineru-section.tsx b/src/components/settings/sections/mineru-section.tsx
index 638295a9..5dcee41d 100644
--- a/src/components/settings/sections/mineru-section.tsx
+++ b/src/components/settings/sections/mineru-section.tsx
@@ -20,11 +20,13 @@ export function MineruSection({ draft, setDraft }: Props) {
const [testError, setTestError] = useState("")
const handleTest = async () => {
- if (!draft.mineruToken.trim()) return
+ if (draft.mineruBackend === "cloud" && !draft.mineruToken.trim()) return
setTestState("running")
setTestError("")
try {
- await testMineruConnection(draft.mineruToken.trim())
+ await testMineruConnection(draft.mineruToken.trim(), {
+ backend: draft.mineruBackend,
+ })
setTestState("success")
} catch (err) {
setTestState("failed")
@@ -67,23 +69,74 @@ export function MineruSection({ draft, setDraft }: Props) {
{draft.mineruEnabled && (
<div className="space-y-4 pl-1">
<div className="space-y-2">
- <Label htmlFor="mineru-token">
- {t("settings.sections.mineru.token", { defaultValue: "API Token" })}
- </Label>
- <Input
- id="mineru-token"
- type="password"
- value={draft.mineruToken}
- onChange={(e) => {
- setDraft("mineruToken", e.target.value)
- setTestState("idle")
- }}
- placeholder={t("settings.sections.mineru.tokenHint", {
- defaultValue: "Get your token from mineru.net",
- })}
- />
+ <Label>{t("settings.sections.mineru.backend", { defaultValue: "Backend" })}</Label>
+ <div className="space-y-1 pl-1">
+ <label className="flex items-center gap-2">
+ <input
+ type="radio"
+ checked={draft.mineruBackend === "cloud"}
+ onChange={() => {
+ setDraft("mineruBackend", "cloud")
+ setTestState("idle")
+ }}
+ className="h-4 w-4"
+ />
+ <span className="text-sm">
+ {t("settings.sections.mineru.cloudBackend", {
+ defaultValue: "Cloud (mineru.net) - requires token, higher quality",
+ })}
+ </span>
+ </label>
+ <label className="flex items-center gap-2">
+ <input
+ type="radio"
+ checked={draft.mineruBackend === "local"}
+ onChange={() => {
+ setDraft("mineruBackend", "local")
+ setTestState("idle")
+ }}
+ className="h-4 w-4"
+ />
+ <span className="text-sm">
+ {t("settings.sections.mineru.localBackend", {
+ defaultValue: "Local (127.0.0.1:8790) - private, offline",
+ })}
+ </span>
+ </label>
+ </div>
</div>
+ {draft.mineruBackend === "cloud" && (
+ <div className="space-y-2">
+ <Label htmlFor="mineru-token">
+ {t("settings.sections.mineru.token", { defaultValue: "API Token" })}
+ </Label>
+ <Input
+ id="mineru-token"
+ type="password"
+ value={draft.mineruToken}
+ onChange={(e) => {
+ setDraft("mineruToken", e.target.value)
+ setTestState("idle")
+ }}
+ placeholder={t("settings.sections.mineru.tokenHint", {
+ defaultValue: "Get your token from mineru.net",
+ })}
+ />
+ </div>
+ )}
+
+ {draft.mineruBackend === "local" && (
+ <div className="rounded-md border border-blue-500/30 bg-blue-500/10 px-3 py-2">
+ <p className="text-xs text-blue-800 dark:text-blue-200">
+ {t("settings.sections.mineru.localInfo", {
+ defaultValue:
+ "Using local MinerU service on http://127.0.0.1:8790. Make sure the service is running!",
+ })}
+ </p>
+ </div>
+ )}
+
<div className="space-y-2">
<Label htmlFor="mineru-model">
{t("settings.sections.mineru.model", { defaultValue: "Model Version" })}
@@ -126,7 +179,8 @@ export function MineruSection({ draft, setDraft }: Props) {
size="sm"
onClick={handleTest}
disabled={
- !draft.mineruToken.trim() || testState === "running"
+ (draft.mineruBackend === "cloud" && !draft.mineruToken.trim()) ||
+ testState === "running"
}
>
{testState === "running"
diff --git a/src/components/settings/settings-types.ts b/src/components/settings/settings-types.ts
index dc35229d..7d7e89a4 100644
--- a/src/components/settings/settings-types.ts
+++ b/src/components/settings/settings-types.ts
@@ -78,6 +78,7 @@ export interface SettingsDraft {
// MinerU PDF parsing
mineruEnabled: boolean
+ mineruBackend: "cloud" | "local"
mineruToken: string
mineruModelVersion: MineruModelVersion
diff --git a/src/components/settings/settings-view.tsx b/src/components/settings/settings-view.tsx
index 786c6ece..63a37d5f 100644
--- a/src/components/settings/settings-view.tsx
+++ b/src/components/settings/settings-view.tsx
@@ -158,6 +158,7 @@ function initialDraft(
scheduledImportInterval: scheduledImport.interval,
sourceWatchConfig: normalizeSourceWatchConfig(sourceWatch),
mineruEnabled: mineru.enabled,
+ mineruBackend: mineru.backend || "cloud",
mineruToken: mineru.token,
mineruModelVersion: mineru.modelVersion,
apiEnabled: apiConfig.enabled,
@@ -395,6 +396,7 @@ export function SettingsView() {
}
const newMineruConfig = {
enabled: draft.mineruEnabled,
+ backend: draft.mineruBackend,
token: draft.mineruToken.trim(),
modelVersion: draft.mineruModelVersion,
}
diff --git a/src/lib/mineru.ts b/src/lib/mineru.ts
index 242256c6..187f6c58 100644
--- a/src/lib/mineru.ts
+++ b/src/lib/mineru.ts
@@ -6,6 +6,7 @@ import { getFileName, normalizePath } from "@/lib/path-utils"
import type { SavedImage } from "@/lib/extract-source-images"
const API_BASE = "https://mineru.net/api/v4"
+const LOCAL_API_BASE = "http://127.0.0.1:8790/api/mineru-local"
const POLL_INTERVAL_MS = 3_000
const POLL_TIMEOUT_MS = 300_000 // 5 minutes
const MAX_ACCURATE_PARSE_BYTES = 200 * 1024 * 1024
@@ -621,6 +622,68 @@ async function downloadAndExtractMarkdown(
}
}
+// ── Local backend ──
+
+/**
+ * Parse a document using a self-hosted MinerU service (no token required).
+ * The service accepts a JSON body with base64 file content and exposes
+ * task polling + result endpoints. Selected via `config.backend === "local"`.
+ */
+async function parseWithLocalMineru(
+ sourcePath: string,
+ fileName: string,
+ onProgress?: (msg: string) => void,
+ signal?: AbortSignal,
+): Promise<string> {
+ const httpFetch = await getHttpFetch()
+ const { base64 } = await readFileAsBase64(sourcePath)
+
+ onProgress?.("Uploading to local MinerU...")
+ const submitRes = await httpFetch(`${LOCAL_API_BASE}/parse`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ signal,
+ body: JSON.stringify({ file: base64, filename: fileName }),
+ })
+ if (!submitRes.ok) {
+ const text = await submitRes.text().catch(() => "")
+ throw new Error(`Local MinerU submit failed: HTTP ${submitRes.status}: ${text}`)
+ }
+
+ const submitData = await submitRes.json()
+ const taskId = submitData.task_id
+ if (!taskId) throw new Error("Local MinerU returned no task ID")
+
+ onProgress?.("Waiting for local MinerU to finish...")
+ const start = Date.now()
+ while (Date.now() - start < POLL_TIMEOUT_MS) {
+ throwIfAborted(signal)
+
+ const statusRes = await httpFetch(`${LOCAL_API_BASE}/tasks/${taskId}`, { signal })
+ if (!statusRes.ok) {
+ throw new Error(`Local MinerU status check failed: HTTP ${statusRes.status}`)
+ }
+ const status = await statusRes.json()
+
+ if (status.status === "done") {
+ onProgress?.("Downloading parsed result...")
+ const resultRes = await httpFetch(`${LOCAL_API_BASE}/results/${taskId}`, { signal })
+ if (!resultRes.ok) {
+ throw new Error(`Local MinerU download failed: HTTP ${resultRes.status}`)
+ }
+ const result = await resultRes.json()
+ return result.content || ""
+ }
+ if (status.status === "failed") {
+ throw new Error(`Local MinerU parsing failed: ${status.error || "unknown error"}`)
+ }
+
+ await waitForPollInterval(signal)
+ }
+
+ throw new Error("Local MinerU parsing timed out")
+}
+
// ── Public API ──
/**
@@ -652,6 +715,14 @@ export async function parseWithMineruResult(
assetOptions?: MineruAssetOptions,
): Promise<MineruExtractedMarkdown> {
throwIfAborted(signal)
+
+ if (config.backend === "local") {
+ const fileName = sourcePath.split("/").pop() ?? "document.pdf"
+ const markdown = await parseWithLocalMineru(sourcePath, fileName, onProgress, signal)
+ onProgress?.("Done")
+ return { markdown, savedImages: [] }
+ }
+
if (!config.token) throw new Error("MinerU API token not configured")
if (config.modelVersion !== "pipeline" && config.modelVersion !== "vlm") {
throw new Error("MinerU PDF parsing supports only pipeline or vlm model versions")
@@ -696,11 +767,26 @@ export async function parseWithMineruResult(
}
/**
- * Test MinerU API connectivity by submitting a minimal task.
- * Returns true if the token is valid.
+ * Test MinerU connectivity.
+ *
+ * Cloud backend: submits a minimal task to validate the token.
+ * Local backend: checks the local service health endpoint (no token needed).
*/
-export async function testMineruConnection(token: string): Promise<void> {
+export async function testMineruConnection(
+ token: string,
+ config?: Pick<MineruConfig, "backend">,
+): Promise<void> {
const httpFetch = await getHttpFetch()
+
+ if (config?.backend === "local") {
+ const res = await httpFetch(`${LOCAL_API_BASE}/health`)
+ if (!res.ok) {
+ const text = await res.text().catch(() => "")
+ throw new Error(`Local MinerU service unavailable: HTTP ${res.status}: ${text}`)
+ }
+ return
+ }
+
const res = await httpFetch(`${API_BASE}/extract/task`, {
method: "POST",
headers: await mineruHeaders(token),
diff --git a/src/stores/wiki-store.ts b/src/stores/wiki-store.ts
index e237158e..b6b027de 100644
--- a/src/stores/wiki-store.ts
+++ b/src/stores/wiki-store.ts
@@ -257,6 +257,8 @@ export type MineruModelVersion = "pipeline" | "vlm"
export interface MineruConfig {
enabled: boolean
+ /** Parsing backend: MinerU cloud API (default) or a self-hosted local service. */
+ backend?: "cloud" | "local"
token: string
modelVersion: MineruModelVersion
}