Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(doclib): enforce private credential ACLs on macOS
myhlolifd4e4133e963
diff --git a/docs/next/decisions/0035-doclib-persistent-token.md b/docs/next/decisions/0035-doclib-persistent-token.mdindex a08cc2dc..45852c6c 100644--- a/docs/next/decisions/0035-doclib-persistent-token.md+++ b/docs/next/decisions/0035-doclib-persistent-token.md@@ -19,7 +19,8 @@ TCP loopback 仅限制连接来源机器。读取正文、导出到指定路径- 全部 Doclib HTTP 接口使用 Bearer 认证,UDS/TCP 一致;状态、shutdown、OpenAPI 不例外。在业务路由执行前拒绝缺失、重复或错误的 Authorization,不提供匿名降级。- 配置正文写入前限制临时文件权限,再原子替换。POSIX 使用 `0600`,Windows 使用仅授权当前账户- 与 SYSTEM 的受保护 DACL,并读回验证;父目录不能让其他普通账户替换文件。无法落实权限时失败。+ 与 SYSTEM 的受保护 DACL;macOS 同时清除扩展 ACL,并在创建时禁止继承 ACL。设置后读回验证,+ 父目录的权限位及平台 ACL 不能让其他普通账户替换文件。无法落实权限时失败。- CLI/SDK 默认发现重新读取当前配置中的 token;显式地址必须同时提供显式 token。endpoint 升为 version 3,不存 token;`server_id` 继续每次启动变化。旧 version 1/2 仅保留诊断读取,SDK 返回 `doclib_upgrade_required`,升级前须先用旧 CLI 停止旧服务。diff --git a/docs/next/sdk/doclib-client.md b/docs/next/sdk/doclib-client.mdindex 346fe8b2..fa6ddc28 100644--- a/docs/next/sdk/doclib-client.md+++ b/docs/next/sdk/doclib-client.md@@ -105,8 +105,9 @@ finally:凭证,轮换后应重新构造。首次升级到认证版本前,应先用旧版本 CLI 停止旧服务。凭证授予当前文档库的完整访问权限,共用凭证的程序属于同一个可信范围。配置文件使用 POSIX `0600`-或 Windows 受保护 DACL,普通其他账户不可读取;父目录也不能允许其他普通账户替换文件。权限设置或-验证失败时拒绝启动。建议继续使用 loopback,不将本机文档库直接暴露给不可信网络。+或 Windows 受保护 DACL;macOS 还会清除文件的扩展 ACL,并在创建临时文件时禁止继承 ACL。+父目录的权限位及平台 ACL 都不能允许其他普通账户替换文件。权限设置或验证失败时拒绝启动。+建议继续使用 loopback,不将本机文档库直接暴露给不可信网络。## Parse 方法diff --git a/mineru/utils/_private_file_macos.py b/mineru/utils/_private_file_macos.pynew file mode 100644index 00000000..b9a991ef--- /dev/null+++ b/mineru/utils/_private_file_macos.py@@ -0,0 +1,177 @@+"""使用 macOS 原生 ACL 接口保护凭证,避免仅检查 BSD 权限位遗漏额外授权。"""++from __future__ import annotations++import ctypes+import errno+import os+import stat+from pathlib import Path++_ACL_TYPE_EXTENDED = 0x100+_ACL_EXTENDED_ALLOW = 1+_ACL_EXTENDED_DENY = 2+_ACL_ENTRY_ONLY_INHERIT = 1 << 8+_ACL_FLAG_NO_INHERIT = 1 << 17+# 目录只允许列举、搜索、读取属性/扩展属性/安全信息及同步;其他或未知权限均按可写处理。+_READ_ONLY_PERMISSIONS = (1 << 1) | (1 << 3) | (1 << 7) | (1 << 9) | (1 << 11) | (1 << 20)+++def _load_security_library() -> ctypes.CDLL:+ """按需加载系统 ACL API,并显式声明指针宽度,避免 arm64 上地址截断。"""+ library = ctypes.CDLL("/usr/lib/libSystem.B.dylib", use_errno=True)+ pointer = ctypes.c_void_p+ library.acl_init.argtypes = [ctypes.c_int]+ library.acl_init.restype = pointer+ library.acl_free.argtypes = [pointer]+ library.acl_free.restype = ctypes.c_int+ library.acl_get_fd_np.argtypes = [ctypes.c_int, ctypes.c_int]+ library.acl_get_fd_np.restype = pointer+ library.acl_get_entry.argtypes = [pointer, ctypes.c_int, ctypes.POINTER(pointer)]+ library.acl_get_entry.restype = ctypes.c_int+ library.acl_get_tag_type.argtypes = [pointer, ctypes.POINTER(ctypes.c_int)]+ library.acl_get_tag_type.restype = ctypes.c_int+ library.acl_get_permset_mask_np.argtypes = [pointer, ctypes.POINTER(ctypes.c_uint64)]+ library.acl_get_permset_mask_np.restype = ctypes.c_int+ library.acl_get_flagset_np.argtypes = [pointer, ctypes.POINTER(pointer)]+ library.acl_get_flagset_np.restype = ctypes.c_int+ library.acl_get_flag_np.argtypes = [pointer, ctypes.c_uint]+ library.acl_get_flag_np.restype = ctypes.c_int+ library.acl_add_flag_np.argtypes = [pointer, ctypes.c_uint]+ library.acl_add_flag_np.restype = ctypes.c_int+ library.acl_get_qualifier.argtypes = [pointer]+ library.acl_get_qualifier.restype = pointer+ library.mbr_uid_to_uuid.argtypes = [ctypes.c_uint, pointer]+ library.mbr_uid_to_uuid.restype = ctypes.c_int+ library.filesec_init.argtypes = []+ library.filesec_init.restype = pointer+ library.filesec_free.argtypes = [pointer]+ library.filesec_free.restype = None+ library.filesec_set_property.argtypes = [pointer, ctypes.c_int, pointer]+ library.filesec_set_property.restype = ctypes.c_int+ library.openx_np.argtypes = [ctypes.c_char_p, ctypes.c_int, pointer]+ library.openx_np.restype = ctypes.c_int+ library.fchmodx_np.argtypes = [ctypes.c_int, pointer]+ library.fchmodx_np.restype = ctypes.c_int+ return library+++def _checked(result: int | None) -> int:+ """保留原生错误码,权限或文件系统能力不足时不得静默降级。"""+ if result is None or result == -1:+ code = ctypes.get_errno()+ raise OSError(code, os.strerror(code))+ return result+++def _check_acl(library: ctypes.CDLL, fd: int, *, directory: bool) -> None:+ """校验已打开对象的 ACL;目录拒绝不可信写权限,凭证文件必须没有扩展授权。"""+ acl = library.acl_get_fd_np(fd, _ACL_TYPE_EXTENDED)+ # Darwin 对有效 fd 上不存在的扩展 ACL 返回 ENOENT;其他错误不能视为没有 ACL。+ if acl is None and ctypes.get_errno() == errno.ENOENT:+ return+ _checked(acl)+ try:+ entry = ctypes.c_void_p()+ entry_id = 0 # ACL_FIRST_ENTRY;Darwin 的 ACL_NEXT_ENTRY 为 -1。+ trusted: set[bytes] = set()+ while True:+ result = library.acl_get_entry(acl, entry_id, ctypes.byref(entry))+ # Darwin 用 -1/EINVAL 表示迭代结束,与 Linux 的返回约定不同。+ if result == -1 and ctypes.get_errno() == errno.EINVAL:+ break+ _checked(result)+ entry_id = -1+ if not directory:+ raise PermissionError("The filesystem cannot enforce private credential configuration ACLs.")+ tag, flags, permissions = ctypes.c_int(), ctypes.c_void_p(), ctypes.c_uint64()+ _checked(library.acl_get_tag_type(entry, ctypes.byref(tag)))+ _checked(library.acl_get_flagset_np(entry, ctypes.byref(flags)))+ inherit_only = _checked(library.acl_get_flag_np(flags, _ACL_ENTRY_ONLY_INHERIT))+ if tag.value == _ACL_EXTENDED_DENY or inherit_only:+ continue+ _checked(library.acl_get_permset_mask_np(entry, ctypes.byref(permissions)))+ if tag.value == _ACL_EXTENDED_ALLOW and not permissions.value & ~_READ_ONLY_PERMISSIONS:+ continue+ if not trusted:+ for uid in {os.getuid(), 0}:+ identifier = ctypes.create_string_buffer(16)+ code = library.mbr_uid_to_uuid(uid, identifier)+ if code:+ raise OSError(code, os.strerror(code))+ trusted.add(identifier.raw)+ qualifier = _checked(library.acl_get_qualifier(entry))+ try:+ if tag.value != _ACL_EXTENDED_ALLOW or ctypes.string_at(qualifier, 16) not in trusted:+ raise PermissionError("Credential configuration directory must not be writable by other users.")+ finally:+ library.acl_free(qualifier)+ finally:+ library.acl_free(acl)+++def _check_file_owner(fd: int) -> None:+ """用实际文件描述符复核类型、硬链接数量和所有者,避免只信任路径预检。"""+ info = os.fstat(fd)+ if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:+ raise PermissionError("Credential configuration must be a regular file without links.")+ if info.st_uid != os.getuid():+ raise PermissionError("Credential configuration must be owned by the current user.")+++def protect_macos_file(path: Path, *, create: bool = False) -> int | None:+ """原子设置 0600 和禁止继承的空 ACL;创建文件时从首个可见时刻即保持私密。"""+ library = _load_security_library()+ parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)+ acl, descriptor = None, None+ fd = None+ created = False+ transferred = False+ try:+ parent = os.fstat(parent_fd)+ if parent.st_uid not in {os.getuid(), 0}:+ raise PermissionError("Credential configuration directory must have a trusted owner.")+ if parent.st_mode & 0o022:+ raise PermissionError("Credential configuration directory must not be writable by other users.")+ _check_acl(library, parent_fd, directory=True)+ acl = _checked(library.acl_init(0))+ flags = ctypes.c_void_p()+ _checked(library.acl_get_flagset_np(acl, ctypes.byref(flags)))+ _checked(library.acl_add_flag_np(flags, _ACL_FLAG_NO_INHERIT))+ descriptor = _checked(library.filesec_init())+ mode, acl_pointer = ctypes.c_ushort(0o600), ctypes.c_void_p(acl)+ # FILESEC_MODE=4、FILESEC_ACL=5;属性值分别为 mode_t 和 acl_t 的地址。+ _checked(library.filesec_set_property(descriptor, 4, ctypes.byref(mode)))+ _checked(library.filesec_set_property(descriptor, 5, ctypes.byref(acl_pointer)))+ if create:+ fd = _checked(+ library.openx_np(+ os.fsencode(path), os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_NOFOLLOW | os.O_CLOEXEC, descriptor+ )+ )+ created = True+ else:+ fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)+ _check_file_owner(fd)+ # 创建时已禁止 ACL 继承;此处也补回被严格 umask 去掉的所有者读写位。+ _checked(library.fchmodx_np(fd, descriptor))+ if stat.S_IMODE(os.fstat(fd).st_mode) != 0o600:+ raise PermissionError("The filesystem cannot enforce private credential configuration permissions.")+ _check_acl(library, fd, directory=False)+ if create:+ transferred = True+ return fd+ return None+ finally:+ if fd is not None and not transferred:+ os.close(fd)+ if created and not transferred:+ path.unlink(missing_ok=True)+ if descriptor is not None:+ library.filesec_free(descriptor)+ if acl is not None:+ library.acl_free(acl)+ os.close(parent_fd)+++__all__ = ["protect_macos_file"]diff --git a/mineru/utils/private_file.py b/mineru/utils/private_file.pyindex 2847ad02..226bf344 100644--- a/mineru/utils/private_file.py+++ b/mineru/utils/private_file.py@@ -1,10 +1,11 @@-"""为本机凭证配置提供 POSIX 权限和 Windows DACL 保护。"""+"""为本机凭证配置提供 POSIX 权限、macOS ACL 和 Windows DACL 保护。"""from __future__ import annotationsimport osimport secretsimport stat+import sysimport tempfilefrom pathlib import Path@@ -18,6 +19,11 @@ def protect_private_file(path: str | Path) -> None:if os.name == "nt":_protect_windows_file(path)return+ if sys.platform == "darwin":+ from ._private_file_macos import protect_macos_file++ protect_macos_file(path)+ returnif info.st_uid != os.getuid():raise PermissionError("Credential configuration must be owned by the current user.")parent = path.parent.stat()@@ -32,7 +38,7 @@ def protect_private_file(path: str | Path) -> None:def create_private_tempfile(directory: str) -> tuple[int, str]:"""从创建时就限制凭证临时文件权限,防止先打开的其他账户句柄读取后写入的正文。"""- if os.name != "nt":+ if os.name != "nt" and sys.platform != "darwin":fd, path = tempfile.mkstemp(prefix=".config.", suffix=".tmp", dir=directory)try:protect_private_file(path)@@ -44,7 +50,12 @@ def create_private_tempfile(directory: str) -> tuple[int, str]:for _ in range(10):path = Path(directory).absolute() / f".config.{secrets.token_hex(12)}.tmp"try:- fd = _protect_windows_file(path, create=True)+ if sys.platform == "darwin":+ from ._private_file_macos import protect_macos_file++ fd = protect_macos_file(path, create=True)+ else:+ fd = _protect_windows_file(path, create=True)except FileExistsError:continueassert fd is not Nonediff --git a/tests/unittest/test_private_file_macos.py b/tests/unittest/test_private_file_macos.pynew file mode 100644index 00000000..cf7c831c--- /dev/null+++ b/tests/unittest/test_private_file_macos.py@@ -0,0 +1,225 @@+"""在真实 macOS 文件系统上验证 BSD 权限之外的扩展 ACL 和创建时保护。"""++from __future__ import annotations++import ctypes+import errno+import os+import stat+import subprocess+import sys+from pathlib import Path++import pytest+import yaml++from mineru import config as configuration+from mineru.config import load_doclib_server_config, update_config_file+from mineru.utils import _private_file_macos as macos+from mineru.utils.private_file import create_private_tempfile, protect_private_file++pytestmark = pytest.mark.skipif(sys.platform != "darwin", reason="Requires real macOS ACL support")+++def _add_acl(path: Path, entry: str) -> None:+ """用系统 chmod 构造真实 ACL,独立于被测的 ctypes 实现。"""+ subprocess.run(["/bin/chmod", "+a", entry, str(path)], check=True, capture_output=True)+++def _acl_entries(path: Path) -> list[str]:+ """通过系统 ls 读取真实 ACL,避免验证复用生产代码导致同源误判。"""+ result = subprocess.run(["/bin/ls", "-lde", str(path)], check=True, capture_output=True, text=True)+ return result.stdout.splitlines()[1:]+++def test_existing_public_acl_is_removed_and_protection_is_idempotent(tmp_path: Path) -> None:+ """0600 不能屏蔽 Everyone ACL;保护后必须清除额外授权并保留正文。"""+ path = tmp_path / "配置.yaml"+ path.write_text("fake credential", encoding="utf-8")+ path.chmod(0o600)+ _add_acl(path, "everyone allow read,write,append,delete,writesecurity,chown")+ assert _acl_entries(path)+ for _ in range(2):+ protect_private_file(path)+ assert stat.S_IMODE(path.stat().st_mode) == 0o600+ assert _acl_entries(path) == []+ assert path.read_text(encoding="utf-8") == "fake credential"++[email protected]("create", [False, True])+ "permission",+ ["add_file", "add_subdirectory", "delete_child", "delete", "writeattr", "writeextattr", "writesecurity", "chown"],+)+def test_unsafe_directory_acl_is_rejected(tmp_path: Path, permission: str, create: bool) -> None:+ """即使目录为 0700,额外授予其他账户的写入或接管权限也必须拒绝。"""+ path = tmp_path / "config.yaml"+ path.write_text("unchanged", encoding="utf-8")+ tmp_path.chmod(0o700)+ _add_acl(tmp_path, f"everyone allow {permission}")+ before = _acl_entries(tmp_path)+ with pytest.raises(PermissionError, match="directory must not be writable"):+ if create:+ create_private_tempfile(str(tmp_path))+ else:+ protect_private_file(path)+ assert _acl_entries(tmp_path) == before+ assert path.read_text(encoding="utf-8") == "unchanged"+ assert not list(tmp_path.glob(".config.*.tmp"))+++ "entry",+ [+ "everyone allow list,search,readattr,readextattr,readsecurity",+ "everyone deny delete",+ "everyone allow read,write,file_inherit,only_inherit",+ "user:root allow add_file,delete_child,writesecurity",+ "current-user",+ ],+)+def test_safe_directory_acls_are_preserved(tmp_path: Path, entry: str) -> None:+ """只读、拒绝、仅继承和可信用户权限不能像 Windows 的 OWNER RIGHTS 一样被误拒。"""+ if entry == "current-user":+ import pwd++ entry = f"user:{pwd.getpwuid(os.getuid()).pw_name} allow add_file,delete_child,writesecurity"+ _add_acl(tmp_path, entry)+ before = _acl_entries(tmp_path)+ fd, filename = create_private_tempfile(str(tmp_path))+ try:+ path = Path(filename)+ assert not os.get_inheritable(fd)+ assert _acl_entries(path) == []+ os.write(fd, b"private content")+ protect_private_file(path)+ assert path.read_bytes() == b"private content"+ assert _acl_entries(path) == []+ assert _acl_entries(tmp_path) == before+ finally:+ os.close(fd)+++def test_tempfile_has_no_inherited_acl_at_creation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """在原生创建调用刚返回时检查 ACL,防止先公开创建再清理的句柄抢占窗口。"""+ _add_acl(tmp_path, "everyone allow read,write,file_inherit,only_inherit")+ library = macos._load_security_library()+ native_open = library.openx_np+ observed: list[list[str]] = []++ def observed_open(path: bytes, flags: int, descriptor: int) -> int:+ """记录 openx_np 返回瞬间的实际 ACL,而不是只检查函数最终结果。"""+ fd = native_open(path, flags, descriptor)+ if fd >= 0:+ observed.append(_acl_entries(Path(os.fsdecode(path))))+ return fd++ monkeypatch.setattr(library, "openx_np", observed_open)+ monkeypatch.setattr(macos, "_load_security_library", lambda: library)+ fd, filename = create_private_tempfile(str(tmp_path))+ try:+ assert observed == [[]]+ assert not os.get_inheritable(fd)+ assert Path(filename).read_bytes() == b""+ assert stat.S_IMODE(os.fstat(fd).st_mode) == 0o600+ finally:+ os.close(fd)+++def test_first_start_restart_and_atomic_update_keep_private_acl(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """新 home 继承只读 ACL 后仍能启动,重启和原子更新始终保持同一私密凭证。"""+ _add_acl(tmp_path, "everyone allow read,file_inherit,directory_inherit,only_inherit")+ home = tmp_path / "新配置目录"+ monkeypatch.setenv("MINERU_HOME", str(home))+ monkeypatch.delenv("MINERU_CONFIG", raising=False)+ monkeypatch.delenv("MINERU_DOCLIB_AUTH_TOKEN", raising=False)+ monkeypatch.setattr(configuration, "_loaded_config", None)+ first = load_doclib_server_config()+ token = first.config.doclib.auth.token.get_secret_value()+ path = home / "config.yaml"+ assert _acl_entries(home)+ assert _acl_entries(path) == []+ assert stat.S_IMODE(path.stat().st_mode) == 0o600+ assert load_doclib_server_config().config.doclib.auth.token.get_secret_value() == token+ update_config_file({"log": {"level": "debug"}})+ assert _acl_entries(path) == []+ assert stat.S_IMODE(path.stat().st_mode) == 0o600+ raw = yaml.safe_load(path.read_text(encoding="utf-8"))+ assert raw["doclib"]["auth"]["token"] == token+ assert raw["log"]["level"] == "debug"+ assert not list(home.glob(".config.*.tmp"))+++def test_strict_umask_does_not_prevent_private_tempfile_creation(tmp_path: Path) -> None:+ """严格 umask 只会减少创建时权限,返回前应补回所有者读写位。"""+ previous = os.umask(0o777)+ try:+ fd, filename = create_private_tempfile(str(tmp_path))+ finally:+ os.umask(previous)+ try:+ assert stat.S_IMODE(os.fstat(fd).st_mode) == 0o600+ assert _acl_entries(Path(filename)) == []+ os.write(fd, b"private content")+ assert Path(filename).read_bytes() == b"private content"+ finally:+ os.close(fd)++[email protected]("failure", ["read_acl", "create", "set_acl", "verify"])+def test_native_failures_do_not_leave_credential_tempfiles(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str+) -> None:+ """ACL 读取、原子创建或读回验证失败时直接报错,并清理已创建的文件和句柄。"""+ library = macos._load_security_library()+ native_get_acl, native_open, native_chmod = library.acl_get_fd_np, library.openx_np, library.fchmodx_np+ created_fds: list[int] = []++ def failing_acl(fd: int, acl_type: int) -> int | None:+ """模拟文件系统拒绝读取父目录 ACL 或刚创建文件的 ACL。"""+ if failure == "read_acl" or fd in created_fds:+ ctypes.set_errno(errno.EACCES)+ return None+ return native_get_acl(fd, acl_type)++ def tracked_open(path: bytes, flags: int, descriptor: int) -> int:+ """记录真实句柄或模拟不支持原子 ACL 创建的文件系统。"""+ if failure == "create":+ ctypes.set_errno(errno.ENOTSUP)+ return -1+ fd = native_open(path, flags, descriptor)+ if fd >= 0:+ created_fds.append(fd)+ return fd++ def failing_chmod(fd: int, descriptor: int) -> int:+ """模拟无法落实最终权限的文件系统,检查返回句柄前的失败清理。"""+ if failure == "set_acl":+ ctypes.set_errno(errno.EACCES)+ return -1+ return native_chmod(fd, descriptor)++ monkeypatch.setattr(library, "acl_get_fd_np", failing_acl)+ monkeypatch.setattr(library, "openx_np", tracked_open)+ monkeypatch.setattr(library, "fchmodx_np", failing_chmod)+ monkeypatch.setattr(macos, "_load_security_library", lambda: library)+ with pytest.raises(OSError):+ create_private_tempfile(str(tmp_path))+ assert not list(tmp_path.glob(".config.*.tmp"))+ for fd in created_fds:+ with pytest.raises(OSError, match="Bad file descriptor"):+ os.fstat(fd)++[email protected]("link", ["hard", "symbolic"])+def test_linked_credentials_are_rejected(tmp_path: Path, link: str) -> None:+ """ACL 保护不能绕过已有的符号链接和硬链接拒绝规则。"""+ original, path = tmp_path / "original", tmp_path / "config.yaml"+ original.write_text("unchanged", encoding="utf-8")+ if link == "hard":+ path.hardlink_to(original)+ else:+ path.symlink_to(original)+ with pytest.raises(PermissionError, match="regular file without links"):+ protect_private_file(path)+ assert original.read_text(encoding="utf-8") == "unchanged"