Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

23.4 KB
Raw Blame
"use client";

import { useSettings } from "@/features/settings/store/SettingsStore";
import { useStagedSettings } from "@/features/settings/store/useStagedSettings";
import { useEffect, useRef, useState } from "react";
import { CheckCircle2, Eye, EyeOff, Loader2, XCircle } from "lucide-react";
import { useTranslation } from "react-i18next";

import {
  SettingRow,
  SettingSection,
  inputClass,
  nativeSelectClass,
  selectOptionClass,
} from "@/components/settings/shared";
import { Toggle } from "@/components/settings/Toggle";
import { apiFetch, apiUrl } from "@/lib/api";

type MinerUMode = "local" | "cloud";
type MinerUModelVersion = "pipeline" | "vlm";
type MinerUDownloadSource = "huggingface" | "modelscope";
type MinerUDownloadType = "pipeline" | "vlm" | "all";

const DEFAULT_BASE_URL = "https://mineru.net";
const DEFAULT_HF_ENDPOINT = "https://huggingface.co";
const LANGUAGE_AUTO = "auto";
const TOKEN_MASK = "••••••••••••";
const MODEL_VERSIONS: MinerUModelVersion[] = ["pipeline", "vlm"];
const DOWNLOAD_SOURCES: MinerUDownloadSource[] = ["huggingface", "modelscope"];
const DOWNLOAD_SOURCE_LABELS: Record<MinerUDownloadSource, string> = {
  huggingface: "HuggingFace",
  modelscope: "ModelScope",
};
const DOWNLOAD_TYPES: MinerUDownloadType[] = ["pipeline", "vlm", "all"];

type MinerUSettings = {
  api_token?: string;
  mode: MinerUMode;
  api_base_url: string;
  local_cli_path: string;
  model_download_source: MinerUDownloadSource;
  model_download_endpoint: string;
  model_version: MinerUModelVersion;
  language: string;
  enable_formula: boolean;
  enable_table: boolean;
  is_ocr: boolean;
  allow_local_model_download: boolean;
};

type DownloadStatus = {
  state: "running" | "done" | "failed" | "cancelled" | string;
  lines: string[];
  message: string;
};

type MinerUPayload = {
  settings: MinerUSettings & { version?: number };
  api_token_set: boolean;
  local_cli?: {
    found: boolean;
    command: string;
    path: string;
    source?: "configured" | "path";
  };
};

function normalizeDraft(payload: MinerUPayload): MinerUSettings {
  const s = payload.settings;
  return {
    mode: s.mode === "cloud" ? "cloud" : "local",
    api_base_url: (s.api_base_url || "https://mineru.net").replace(/\/+$/, ""),
    local_cli_path: s.local_cli_path || "",
    model_download_source:
      s.model_download_source === "modelscope" ? "modelscope" : "huggingface",
    model_download_endpoint: (s.model_download_endpoint || "").replace(/\/+$/, ""),
    model_version: s.model_version === "vlm" ? "vlm" : "pipeline",
    language: s.language || "auto",
    enable_formula: Boolean(s.enable_formula),
    enable_table: Boolean(s.enable_table),
    is_ocr: Boolean(s.is_ocr),
    allow_local_model_download: Boolean(s.allow_local_model_download),
  };
}

export function MinerUEngineSettings() {
  const { t } = useTranslation();
  const { draftRevision } = useSettings();
  const [payload, setPayload] = useState<MinerUPayload | null>(null);
  const [liveDraft, setLiveDraft] = useState<MinerUSettings | null>(null);
  const [draft, setDraft] = useStagedSettings("mineru", liveDraft, setLiveDraft);
  const tokenDraft = draft?.api_token ?? "";
  const tokenTouched = draft?.api_token !== undefined;
  const [showToken, setShowToken] = useState(false);
  const [loading, setLoading] = useState(true);
  const [testing, setTesting] = useState(false);
  const [testResult, setTestResult] = useState<{
    ok: boolean;
    message: string;
  } | null>(null);
  const [error, setError] = useState<string | null>(null);
  const [download, setDownload] = useState<DownloadStatus | null>(null);
  const [downloadType, setDownloadType] =
    useState<MinerUDownloadType>("pipeline");
  const [startingDownload, setStartingDownload] = useState(false);
  const downloadCursor = useRef(0);

  useEffect(() => {
    let cancelled = false;
    async function load() {
      setLoading(true);
      setError(null);
      try {
        const response = await apiFetch(apiUrl("/api/settings/mineru"));
        const data = (await response.json().catch(() => ({}))) as
          | MinerUPayload
          | { detail?: string };
        if (!response.ok) {
          throw new Error(
            "detail" in data && data.detail
              ? data.detail
              : t("Failed to load MinerU settings."),
          );
        }
        if (cancelled) return;
        const next = data as MinerUPayload;
        setPayload(next);
        setLiveDraft(normalizeDraft(next));
      } catch (err) {
        if (!cancelled)
          setError(err instanceof Error ? err.message : String(err));
      } finally {
        if (!cancelled) setLoading(false);
      }
    }
    load();
    return () => {
      cancelled = true;
    };
  }, [t, draftRevision]);

  function patch(next: Partial<MinerUSettings>) {
    setDraft((current) => (current ? { ...current, ...next } : current));
  }

  async function testConnection() {
    if (!draft) return;
    setTesting(true);
    setTestResult(null);
    setError(null);
    try {
      const body: Record<string, unknown> = {
        ...draft,
        api_token: tokenTouched ? tokenDraft : null,
      };
      const response = await apiFetch(apiUrl("/api/settings/mineru/test"), {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify(body),
      });
      const data = (await response.json().catch(() => ({}))) as {
        ok?: boolean;
        message?: string;
        detail?: string;
      };
      if (!response.ok) {
        throw new Error(data.detail || t("Connection test failed."));
      }
      setTestResult({
        ok: Boolean(data.ok),
        message:
          data.message || (data.ok ? t("OK") : t("Connection test failed.")),
      });
    } catch (err) {
      setTestResult({
        ok: false,
        message: err instanceof Error ? err.message : String(err),
      });
    } finally {
      setTesting(false);
    }
  }

  // Poll the download job while it runs; the cursor protocol fetches only
  // new log lines each tick.
  useEffect(() => {
    if (download?.state !== "running") return;
    const timer = setInterval(async () => {
      try {
        const response = await apiFetch(
          apiUrl(
            `/api/settings/mineru/models/download/status?cursor=${downloadCursor.current}`,
          ),
        );
        if (!response.ok) return;
        const data = (await response.json()) as {
          state?: string;
          lines?: string[];
          next_cursor?: number;
          message?: string;
        };
        downloadCursor.current = data.next_cursor ?? downloadCursor.current;
        setDownload((current) =>
          current
            ? {
                state: data.state || current.state,
                lines: [...current.lines, ...(data.lines || [])].slice(-100),
                message: data.message || "",
              }
            : current,
        );
      } catch {
        // transient network error — keep polling
      }
    }, 1000);
    return () => clearInterval(timer);
  }, [download?.state]);

  async function startDownload() {
    if (!draft) return;
    setStartingDownload(true);
    try {
      const response = await apiFetch(
        apiUrl("/api/settings/mineru/models/download"),
        {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({
            model_type: downloadType,
            source: draft.model_download_source,
            endpoint: draft.model_download_endpoint,
            local_cli_path: draft.local_cli_path,
          }),
        },
      );
      const data = (await response.json().catch(() => ({}))) as {
        ok?: boolean;
        message?: string;
        detail?: string;
      };
      if (!response.ok || !data.ok) {
        setDownload({
          state: "failed",
          lines: [],
          message: data.message || data.detail || t("Download failed."),
        });
        return;
      }
      downloadCursor.current = 0;
      setDownload({ state: "running", lines: [], message: "" });
    } finally {
      setStartingDownload(false);
    }
  }

  async function cancelDownload() {
    try {
      await apiFetch(apiUrl("/api/settings/mineru/models/download/cancel"), {
        method: "POST",
      });
    } catch {
      // status polling will surface the final state either way
    }
  }

  const tokenSet = payload?.api_token_set ?? false;
  const isCloud = draft?.mode === "cloud";
  const localCli = payload?.local_cli;

  function renderTestControl(label: string) {
    return (
      <div className="flex items-center gap-3">
        {testResult && (
          <span
            className={`inline-flex max-w-[40vw] items-center gap-1 text-[12px] ${
              testResult.ok
                ? "text-emerald-600 dark:text-emerald-400"
                : "text-red-600 dark:text-red-400"
            }`}
          >
            {testResult.ok ? (
              <CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
            ) : (
              <XCircle className="h-3.5 w-3.5 shrink-0" />
            )}
            {testResult.message}
          </span>
        )}
        <button
          type="button"
          onClick={testConnection}
          disabled={testing}
          className="inline-flex items-center gap-1.5 rounded-lg border border-[var(--border)] px-3 py-1.5 text-[12px] font-medium text-[var(--foreground)] transition-opacity hover:opacity-80 disabled:opacity-40"
        >
          {testing && <Loader2 className="h-3 w-3 animate-spin" />}
          {label}
        </button>
      </div>
    );
  }

  if (loading) {
    return (
      <div className="flex items-center gap-2 text-[13px] text-[var(--muted-foreground)]">
        <Loader2 className="h-4 w-4 animate-spin" />
        {t("Loading MinerU settings...")}
      </div>
    );
  }

  if (error && !draft) {
    return (
      <div className="mb-5 rounded-xl border border-red-500/30 bg-red-500/10 px-4 py-3 text-[13px] text-red-600 dark:text-red-300">
        {error}
      </div>
    );
  }

  if (!draft) return null;

  return (
    <>
      {error && (
        <div className="mb-5 rounded-xl border border-red-500/30 bg-red-500/10 px-4 py-3 text-[13px] text-red-600 dark:text-red-300">
          {error}
        </div>
      )}

      <SettingSection
        title={t("Parsing backend")}
        description={t("Choose where PDF parsing runs.")}
      >
        <SettingRow
          title={t("Mode")}
          description={
            isCloud
              ? t("Documents are uploaded to mineru.net for parsing.")
              : t(
                  "Parsing runs on this machine using the local MinerU install.",
                )
          }
          control={
            <div className="inline-flex rounded-lg border border-[var(--border)] p-0.5">
              {(["local", "cloud"] as MinerUMode[]).map((m) => (
                <button
                  key={m}
                  type="button"
                  onClick={() => {
                    patch({ mode: m });
                    setTestResult(null);
                  }}
                  className={`rounded-md px-3 py-1 text-[12px] font-medium transition-colors ${
                    draft.mode === m
                      ? "bg-[var(--foreground)] text-[var(--background)]"
                      : "text-[var(--muted-foreground)] hover:text-[var(--foreground)]"
                  }`}
                >
                  {m === "local" ? t("Local") : t("Cloud API")}
                </button>
              ))}
            </div>
          }
        />
      </SettingSection>

      {!isCloud && (
        <SettingSection
          title={t("Local install")}
          description={t(
            "The MinerU CLI must be reachable by the DeepTutor server process. Reserve >=20 GB of free disk (official guidance): ~2-4 GB install, ~1-2 GB models lazily downloaded on first parse, plus parsing cache.",
          )}
        >
          <SettingRow
            title={
              localCli?.found
                ? t("MinerU CLI detected.")
                : localCli?.source === "configured"
                  ? t("Configured CLI path is not executable.")
                  : t("MinerU CLI not found on PATH.")
            }
            description={
              localCli?.found
                ? localCli.path
                : t(
                    'Install into any Python environment: uv pip install -U "mineru[all]>=3.4.5" — then set the CLI path below, or install into the server environment for PATH auto-detection.',
                  )
            }
            control={renderTestControl(t("Check"))}
          />
          <SettingRow
            title={t("CLI path")}
            description={t(
              "Optional. Point to a mineru executable in an isolated environment (uv tool, pipx, conda) to avoid dependency conflicts. Leave blank to auto-detect from PATH.",
            )}
            control={
              <input
                className={`${inputClass} w-[320px] max-w-[48vw] font-mono text-[12px]`}
                placeholder={t("Auto-detect from PATH")}
                value={draft.local_cli_path}
                onChange={(e) => patch({ local_cli_path: e.target.value })}
              />
            }
          />
        </SettingSection>
      )}

      {!isCloud && (
        <SettingSection
          title={t("Model weights")}
          description={t(
            "Local parsing needs ~1-2 GB of model weights. By default they are NOT downloaded automatically — download them explicitly below, or allow automatic download on first parse.",
          )}
        >
          <SettingRow
            title={t("Allow automatic model download")}
            description={t(
              "Off by default. When off, a local parse fails with guidance instead of silently downloading multi-GB weights on first run. Turn on to let the first parse fetch models, or use the explicit download below.",
            )}
            control={
              <Toggle
                checked={draft.allow_local_model_download}
                onChange={(v) => patch({ allow_local_model_download: v })}
              />
            }
          />
          <SettingRow
            title={t("Download source")}
            description={t("ModelScope is usually faster in mainland China.")}
            control={
              <select
                className={`${nativeSelectClass} w-44`}
                value={draft.model_download_source}
                onChange={(e) =>
                  patch({
                    model_download_source: e.target
                      .value as MinerUDownloadSource,
                  })
                }
              >
                {DOWNLOAD_SOURCES.map((s) => (
                  <option key={s} className={selectOptionClass} value={s}>
                    {DOWNLOAD_SOURCE_LABELS[s]}
                  </option>
                ))}
              </select>
            }
          />
          {draft.model_download_source === "huggingface" && (
            <SettingRow
              title={t("Download address")}
              description={t(
                "Custom HuggingFace endpoint or mirror (e.g. a regional mirror). Leave blank for the official address.",
              )}
              control={
                <input
                  className={`${inputClass} w-[320px] max-w-[48vw] font-mono text-[12px]`}
                  placeholder={DEFAULT_HF_ENDPOINT}
                  value={draft.model_download_endpoint}
                  onChange={(e) =>
                    patch({ model_download_endpoint: e.target.value })
                  }
                />
              }
            />
          )}
          <SettingRow
            title={t("Download models")}
            description={t(
              "Runs mineru-models-download on the server using the source and address above.",
            )}
            control={
              <div className="flex items-center gap-2">
                <select
                  className={`${nativeSelectClass} w-28`}
                  value={downloadType}
                  onChange={(e) =>
                    setDownloadType(e.target.value as MinerUDownloadType)
                  }
                  disabled={download?.state === "running"}
                >
                  {DOWNLOAD_TYPES.map((v) => (
                    <option key={v} className={selectOptionClass} value={v}>
                      {v}
                    </option>
                  ))}
                </select>
                {download?.state === "running" ? (
                  <button
                    type="button"
                    onClick={cancelDownload}
                    className="inline-flex items-center gap-1.5 rounded-lg border border-[var(--border)] px-3 py-1.5 text-[12px] font-medium text-[var(--foreground)] transition-opacity hover:opacity-80"
                  >
                    {t("Cancel")}
                  </button>
                ) : (
                  <button
                    type="button"
                    onClick={startDownload}
                    disabled={startingDownload}
                    className="inline-flex items-center gap-1.5 rounded-lg bg-[var(--foreground)] px-3 py-1.5 text-[12px] font-medium text-[var(--background)] transition-opacity hover:opacity-80 disabled:opacity-40"
                  >
                    {startingDownload && (
                      <Loader2 className="h-3 w-3 animate-spin" />
                    )}
                    {t("Download")}
                  </button>
                )}
              </div>
            }
          />
          {download && (
            <div className="px-1 pb-4">
              <div
                className={`mb-2 inline-flex items-center gap-1.5 text-[12px] ${
                  download.state === "done"
                    ? "text-emerald-600 dark:text-emerald-400"
                    : download.state === "running"
                      ? "text-[var(--muted-foreground)]"
                      : "text-red-600 dark:text-red-400"
                }`}
              >
                {download.state === "running" ? (
                  <Loader2 className="h-3 w-3 animate-spin" />
                ) : download.state === "done" ? (
                  <CheckCircle2 className="h-3.5 w-3.5" />
                ) : (
                  <XCircle className="h-3.5 w-3.5" />
                )}
                {download.state === "running"
                  ? t("Downloading models...")
                  : download.message || download.state}
              </div>
              {download.lines.length > 0 && (
                <pre className="max-h-40 overflow-y-auto whitespace-pre-wrap rounded-lg border border-[var(--border)]/60 bg-[var(--card)] px-3 py-2 font-mono text-[11px] leading-relaxed text-[var(--muted-foreground)]">
                  {download.lines.join("\n")}
                </pre>
              )}
            </div>
          )}
        </SettingSection>
      )}

      {isCloud && (
        <SettingSection
          title={t("Cloud API")}
          description={t("Get an API token from mineru.net → API management.")}
        >
          <SettingRow
            title={t("API base URL")}
            description={t(
              "Override only if you use a self-hosted MinerU endpoint.",
            )}
            control={
              <input
                className={`${inputClass} w-[320px] max-w-[48vw]`}
                placeholder={DEFAULT_BASE_URL}
                value={draft.api_base_url}
                onChange={(e) => patch({ api_base_url: e.target.value })}
              />
            }
          />
          <SettingRow
            title={t("API token")}
            description={
              tokenSet
                ? t(
                    "A token is saved. Type to replace it; leave blank to keep it.",
                  )
                : t("No token saved yet.")
            }
            control={
              <div className="relative inline-flex items-center">
                <input
                  id="mineru-api-token"
                  name="mineru_api_token"
                  type={showToken ? "text" : "password"}
                  autoComplete="new-password"
                  spellCheck={false}
                  data-1p-ignore="true"
                  data-lpignore="true"
                  className={`${inputClass} w-[320px] max-w-[48vw] pr-9 font-mono`}
                  placeholder={tokenSet ? TOKEN_MASK : t("Paste API token")}
                  value={tokenDraft}
                  onChange={(e) => {
                    const val = e.target.value.trim();
                    patch({ api_token: val });
                  }}
                />
                <button
                  type="button"
                  onClick={() => setShowToken((v) => !v)}
                  aria-label={showToken ? t("Hide API token") : t("Show API token")}
                  className="absolute right-1.5 top-1/2 -translate-y-1/2 rounded-md p-1 text-[var(--muted-foreground)] hover:bg-[var(--muted)] hover:text-[var(--foreground)]"
                >
                  {showToken ? (
                    <EyeOff className="h-3.5 w-3.5" />
                  ) : (
                    <Eye className="h-3.5 w-3.5" />
                  )}
                </button>
              </div>
            }
          />
          <SettingRow
            title={t("Test connection")}
            description={t(
              "Verifies the token against the MinerU API (no quota used).",
            )}
            control={renderTestControl(t("Test"))}
          />
        </SettingSection>
      )}

      <SettingSection
        title={t("Parsing options")}
        description={t("Forwarded to MinerU for each document.")}
      >
        <SettingRow
          title={t("Model version")}
          description={t(
            "pipeline is faster; vlm is the vision-language model.",
          )}
          control={
            <select
              className={`${nativeSelectClass} w-40`}
              value={draft.model_version}
              onChange={(e) =>
                patch({ model_version: e.target.value as MinerUModelVersion })
              }
            >
              {MODEL_VERSIONS.map((v) => (
                <option key={v} className={selectOptionClass} value={v}>
                  {v}
                </option>
              ))}
            </select>
          }
        />
        <SettingRow
          title={t("Language")}
          description={t(
            'Document language hint. Use "auto" to let MinerU detect it.',
          )}
          control={
            <input
              className={`${inputClass} w-40`}
              placeholder={LANGUAGE_AUTO}
              value={draft.language}
              onChange={(e) => patch({ language: e.target.value })}
            />
          }
        />
        <SettingRow
          title={t("Extract formulas")}
          control={
            <Toggle
              checked={draft.enable_formula}
              onChange={(v) => patch({ enable_formula: v })}
            />
          }
        />
        <SettingRow
          title={t("Extract tables")}
          control={
            <Toggle
              checked={draft.enable_table}
              onChange={(v) => patch({ enable_table: v })}
            />
          }
        />
        <SettingRow
          title={t("Force OCR")}
          description={t(
            "Treat the PDF as scanned images. Slower; only for non-text PDFs.",
          )}
          control={
            <Toggle
              checked={draft.is_ocr}
              onChange={(v) => patch({ is_ocr: v })}
            />
          }
        />
      </SettingSection>


    </>
  );
}