Skip to content

Turn any document into clean, AI-ready Markdown. Local-first desktop app: reads scanned PDFs, batches folders, runs offline, and uses far fewer tokens than vision models.

15.0 KB
Raw Blame
from __future__ import annotations

import json
import os
import queue
import signal
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path

from .layout import bundled_python_path
from .models import CONVERSION_FIXTURES, FULL_IMPORT_MODULES, CheckResult
from .paths import app_data_root


def _run_python(
    python: Path,
    code: str,
    *,
    cwd: Path | None = None,
    timeout: float = 120.0,
) -> subprocess.CompletedProcess[str]:
    return subprocess.run(
        [str(python), "-I", "-c", code],
        cwd=str(cwd) if cwd else None,
        capture_output=True,
        text=True,
        timeout=timeout,
        check=False,
    )


def verify_bundled_python_version(
    root: Path,
    *,
    platform: str,
    manifest: dict,
) -> CheckResult:
    python = bundled_python_path(root, platform)
    if not python.is_file():
        return CheckResult(
            "bundled-python-version",
            False,
            f"Missing bundled interpreter: {python}",
        )
    expected = manifest.get("python_version")
    proc = _run_python(
        python,
        "import platform; print(platform.python_version())",
    )
    if proc.returncode != 0:
        return CheckResult(
            "bundled-python-version",
            False,
            proc.stderr.strip() or proc.stdout.strip() or "Interpreter failed",
        )
    actual = proc.stdout.strip()
    if actual != expected:
        return CheckResult(
            "bundled-python-version",
            False,
            f"Expected {expected}, got {actual}",
        )
    return CheckResult("bundled-python-version", True, actual)


def verify_full_imports(root: Path, *, platform: str) -> CheckResult:
    python = bundled_python_path(root, platform)
    if not python.is_file():
        return CheckResult(
            "full-runtime-imports",
            False,
            f"Missing bundled interpreter: {python}",
        )
    modules = ", ".join(repr(name) for name in FULL_IMPORT_MODULES)
    code = (
        "import importlib; mods = ["
        + modules
        + "]; "
        + "[importlib.import_module(m) for m in mods]; "
        + "print('imports OK')"
    )
    proc = _run_python(python, code, timeout=180.0)
    if proc.returncode != 0:
        detail = proc.stderr.strip() or proc.stdout.strip() or "Import failed"
        return CheckResult("full-runtime-imports", False, detail)
    return CheckResult("full-runtime-imports", True, "All required modules imported")


def _sidecar_request(
    python: Path,
    sidecar_dir: Path,
    request: dict,
    *,
    timeout: float = 60.0,
) -> tuple[dict | None, str]:
    payload = json.dumps(request) + "\n"
    env = os.environ.copy()
    env["PYTHONPATH"] = str(sidecar_dir)
    # Windows pipe reads are not selectable. Drain stdout on a daemon thread
    # and pass each successive line through a queue so the caller can enforce
    # one deadline without leaving a non-daemon reader behind.
    process_group_options: dict[str, object] = {}
    if os.name == "nt":
        process_group_options["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
    else:
        process_group_options["start_new_session"] = True
    proc = subprocess.Popen(
        [str(python), str(sidecar_dir / "main.py")],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        cwd=str(sidecar_dir),
        env=env,
        **process_group_options,
    )
    stdout_thread: threading.Thread | None = None
    stderr_thread: threading.Thread | None = None

    def terminate_process_tree() -> None:
        if proc.poll() is not None:
            return
        if os.name == "nt":
            subprocess.run(
                ["taskkill", "/PID", str(proc.pid), "/T", "/F"],
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                check=False,
            )
        else:
            try:
                os.killpg(proc.pid, signal.SIGKILL)
            except ProcessLookupError:
                pass

    try:
        if proc.stdin is None or proc.stdout is None or proc.stderr is None:
            return None, "Could not open sidecar subprocess streams"
        proc.stdin.write(payload)
        proc.stdin.flush()
        stdout_lines: queue.Queue[str | None] = queue.Queue()
        stderr_lines: list[str] = []

        def read_stdout() -> None:
            try:
                for output_line in proc.stdout:
                    stdout_lines.put(output_line)
            finally:
                stdout_lines.put(None)

        def read_stderr() -> None:
            stderr_lines.extend(proc.stderr.readlines())

        stdout_thread = threading.Thread(target=read_stdout, daemon=True)
        stderr_thread = threading.Thread(target=read_stderr, daemon=True)
        stdout_thread.start()
        stderr_thread.start()
        deadline = time.monotonic() + timeout
        while True:
            remaining = deadline - time.monotonic()
            if remaining <= 0:
                raise subprocess.TimeoutExpired(proc.args, timeout)
            try:
                line = stdout_lines.get(timeout=remaining)
            except queue.Empty:
                raise subprocess.TimeoutExpired(proc.args, timeout) from None
            if line is None:
                detail = "".join(stderr_lines).strip()
                return None, detail or "No matching IPC response on stdout"
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError:
                continue
            if obj.get("id") == request.get("id") and (
                obj.get("ok") is not None or obj.get("error")
            ):
                return obj, ""
    except subprocess.TimeoutExpired:
        terminate_process_tree()
        raise
    finally:
        # Stop the tree while its root still exists. Closing stdin first lets the
        # sidecar exit and can orphan converter workers that retain the extracted
        # package directory on Windows.
        terminate_process_tree()
        if proc.stdin is not None and not proc.stdin.closed:
            proc.stdin.close()
        try:
            proc.wait(timeout=5)
        except subprocess.TimeoutExpired:
            terminate_process_tree()
            try:
                proc.wait(timeout=5)
            except subprocess.TimeoutExpired:
                pass
        if stdout_thread is not None:
            stdout_thread.join(timeout=1)
        if stderr_thread is not None:
            stderr_thread.join(timeout=1)
        for stream in (proc.stdin, proc.stdout, proc.stderr):
            if stream is not None and not stream.closed:
                stream.close()


def verify_sidecar_health(
    root: Path,
    *,
    platform: str,
    edition: str,
) -> CheckResult:
    sidecar_dir = root / "resources" / "sidecar"
    if edition == "full":
        python = bundled_python_path(root, platform)
    else:
        python = Path(sys.executable)

    request = {"v": 1, "id": "verify-health", "method": "health", "params": {}}
    try:
        response, error = _sidecar_request(python, sidecar_dir, request)
    except subprocess.TimeoutExpired:
        return CheckResult("sidecar-health", False, "Sidecar health request timed out")
    except OSError as exc:
        return CheckResult("sidecar-health", False, str(exc))

    if response is None:
        return CheckResult("sidecar-health", False, error)
    if not response.get("ok"):
        err = response.get("error") or {}
        detail = err.get("detail") or err.get("title") or "Sidecar health failed"
        return CheckResult("sidecar-health", False, str(detail))
    result = response.get("result") or {}
    version = result.get("markitdown_version") or "unknown"
    return CheckResult("sidecar-health", True, f"markitdown={version}")


def verify_conversions(
    root: Path,
    *,
    platform: str,
    edition: str,
    fixtures_dir: Path,
) -> list[CheckResult]:
    sidecar_dir = root / "resources" / "sidecar"
    if edition == "full":
        python = bundled_python_path(root, platform)
    else:
        python = Path(sys.executable)

    results: list[CheckResult] = []
    # Converters are third-party code and some parsers normalize or repair
    # container formats in place. Give each request an isolated copy.
    with tempfile.TemporaryDirectory(prefix="mdflux-fixture-") as scratch:
        scratch_dir = Path(scratch)
        for filename, ext in CONVERSION_FIXTURES:
            sample = fixtures_dir / filename
            if not sample.is_file():
                results.append(CheckResult(
                    f"conversion-fixture-{filename}", False,
                    f"Missing verifier fixture: {sample}", severity="error",
                ))
                continue
            isolated_sample = scratch_dir / filename
            shutil.copy2(sample, isolated_sample)
            request = {
                "v": 1, "id": f"verify-convert-{filename}",
                "method": "convert-one",
                "params": {"path": str(isolated_sample.resolve())},
            }
            try:
                response, error = _sidecar_request(
                    python, sidecar_dir, request, timeout=180.0
                )
            except subprocess.TimeoutExpired:
                results.append(CheckResult(
                    f"conversion-{ext}", False, f"{filename}: timed out"
                ))
                continue
            except OSError as exc:
                results.append(CheckResult(
                    f"conversion-{ext}", False, f"{filename}: {exc}"
                ))
                continue
            if response is None:
                results.append(CheckResult(
                    f"conversion-{ext}", False, f"{filename}: {error}"
                ))
                continue
            if not response.get("ok"):
                err = response.get("error") or {}
                detail = err.get("detail") or err.get("title") or "Conversion failed"
                results.append(CheckResult(
                    f"conversion-{ext}", False, f"{filename}: {detail}"
                ))
                continue
            markdown = (response.get("result") or {}).get("markdown") or ""
            if not str(markdown).strip():
                results.append(CheckResult(
                    f"conversion-{ext}", False,
                    f"{filename}: empty markdown output"
                ))
            else:
                results.append(CheckResult(
                    f"conversion-{ext}", True,
                    f"{filename}: {len(str(markdown))} chars"
                ))
    return results


def verify_ocr_conversions(root: Path, *, platform: str) -> list[CheckResult]:
    """Exercise packaged OCR with both an image and an image-only PDF."""
    python = bundled_python_path(root, platform)
    sidecar_dir = root / "resources" / "sidecar"
    results: list[CheckResult] = []

    with tempfile.TemporaryDirectory(prefix="mdflux-ocr-fixture-") as scratch:
        scratch_dir = Path(scratch).resolve()
        fixture_code = f"""
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont

root = Path({str(scratch_dir)!r})
image = Image.new("RGB", (1400, 360), "white")
font = ImageFont.load_default(size=72)
ImageDraw.Draw(image).text(
    (60, 120),
    "MDFlux OCR release verification 2026",
    fill="black",
    font=font,
    stroke_width=1,
)
image.save(root / "ocr-smoke.png")
image.save(root / "ocr-smoke-scanned.pdf", "PDF", resolution=150.0)
print("fixtures OK")
"""
        generated = _run_python(python, fixture_code, timeout=60.0)
        if generated.returncode != 0:
            detail = generated.stderr.strip() or generated.stdout.strip()
            return [CheckResult("ocr-fixtures", False, detail or "Fixture generation failed")]
        results.append(CheckResult("ocr-fixtures", True, "Image and scanned PDF generated"))

        for filename, check_name in (
            ("ocr-smoke.png", "ocr-image"),
            ("ocr-smoke-scanned.pdf", "ocr-scanned-pdf"),
        ):
            request = {
                "v": 1,
                "id": f"verify-{check_name}",
                "method": "convert-one",
                "params": {"path": str((scratch_dir / filename).resolve())},
            }
            try:
                response, error = _sidecar_request(
                    python, sidecar_dir, request, timeout=300.0
                )
            except subprocess.TimeoutExpired:
                results.append(CheckResult(check_name, False, f"{filename}: timed out"))
                continue
            except OSError as exc:
                results.append(CheckResult(check_name, False, f"{filename}: {exc}"))
                continue

            if response is None:
                results.append(CheckResult(check_name, False, f"{filename}: {error}"))
                continue
            if not response.get("ok"):
                err = response.get("error") or {}
                detail = err.get("detail") or err.get("title") or "OCR failed"
                results.append(CheckResult(check_name, False, f"{filename}: {detail}"))
                continue

            markdown = str((response.get("result") or {}).get("markdown") or "")
            if not markdown.strip():
                results.append(CheckResult(check_name, False, f"{filename}: empty OCR output"))
            else:
                results.append(CheckResult(
                    check_name, True, f"{filename}: {len(markdown)} chars"
                ))

    return results


def _snapshot_provisioning_paths(platform: str) -> dict[str, float | None]:
    root = app_data_root(platform)
    tracked = {
        "app_data_root": root,
        "runtimes": root / "runtimes",
        "active_runtime": root / "active-runtime.json",
    }
    snapshot: dict[str, float | None] = {}
    for key, path in tracked.items():
        if key == "app_data_root":
            continue
        if path.exists():
            snapshot[str(path)] = path.stat().st_mtime
        else:
            snapshot[str(path)] = None
    return snapshot


def verify_no_full_provisioning(
    before: dict[str, float | None],
    *,
    platform: str,
) -> CheckResult:
    after = _snapshot_provisioning_paths(platform)
    created: list[str] = []
    for path, mtime in after.items():
        if path == str(app_data_root(platform)):
            continue
        previous = before.get(path)
        path_obj = Path(path)
        if previous is None and path_obj.exists():
            created.append(path)
        elif previous is not None and path_obj.exists() and path_obj.stat().st_mtime > previous:
            created.append(path)
    if created:
        return CheckResult(
            "full-no-provisioning",
            False,
            "Full verification wrote provisioning state: " + ", ".join(created),
        )
    return CheckResult(
        "full-no-provisioning",
        True,
        "No new Lite provisioning paths were written",
    )


def snapshot_provisioning(platform: str) -> dict[str, float | None]:
    return _snapshot_provisioning_paths(platform)