π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import os
from pathlib import Path
import sys
import tempfile
from typing import Final
import warnings
import cv2
import numpy as np
import onnxruntime as ort
from PIL import Image, ImageOps
import requests
try:
from pillow_heif import register_heif_opener
register_heif_opener()
except Exception:
# HEIF support is optional at import time; decode errors surface later with context.
pass
DEFAULT_MODEL_BASE_URL: Final[str] = "https://models.ente.com/"
DEFAULT_PADDING_RGB: Final[tuple[int, int, int]] = (114, 114, 114)
ALLOW_OPENCV_FALLBACK_ENV: Final[str] = "ML_PARITY_ALLOW_OPENCV_DECODE_FALLBACK"
EXIF_ORIENTATION_TAG: Final[int] = 274
ORIENTATION_TRANSPOSE_METHODS: Final[dict[int, Image.Transpose]] = {
2: Image.Transpose.FLIP_LEFT_RIGHT,
3: Image.Transpose.ROTATE_180,
4: Image.Transpose.FLIP_TOP_BOTTOM,
5: Image.Transpose.TRANSPOSE,
6: Image.Transpose.ROTATE_270,
7: Image.Transpose.TRANSVERSE,
8: Image.Transpose.ROTATE_90,
}
HEIF_EXTENSIONS: Final[frozenset[str]] = frozenset({".heic", ".heif"})
@dataclass(frozen=True)
class ModelArtifact:
file_name: str
path: Path
sha256: str
etag: str | None
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as file:
for chunk in iter(lambda: file.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def ensure_model(
*,
file_name: str,
cache_dir: Path,
base_url: str = DEFAULT_MODEL_BASE_URL,
) -> ModelArtifact:
cache_dir.mkdir(parents=True, exist_ok=True)
model_path = cache_dir / file_name
if model_path.exists() and model_path.stat().st_size > 0:
return ModelArtifact(
file_name=file_name,
path=model_path,
sha256=_sha256_file(model_path),
etag=None,
)
model_url = f"{base_url.rstrip('/')}/{file_name}"
with requests.get(model_url, stream=True, timeout=(10, 300)) as response:
response.raise_for_status()
etag = response.headers.get("ETag")
with tempfile.NamedTemporaryFile(
dir=cache_dir,
prefix=f"{file_name}.",
suffix=".tmp",
delete=False,
) as tmp_file:
tmp_path = Path(tmp_file.name)
for chunk in response.iter_content(chunk_size=1024 * 1024):
if chunk:
tmp_file.write(chunk)
tmp_path.replace(model_path)
return ModelArtifact(
file_name=file_name,
path=model_path,
sha256=_sha256_file(model_path),
etag=etag,
)
def create_ort_session(model_path: Path) -> ort.InferenceSession:
options = ort.SessionOptions()
options.inter_op_num_threads = 1
options.intra_op_num_threads = 1
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
return ort.InferenceSession(
str(model_path),
sess_options=options,
providers=["CPUExecutionProvider"],
)
def l2_normalize_rows(values: np.ndarray) -> np.ndarray:
values = np.asarray(values, dtype=np.float32)
if values.ndim == 1:
norm = float(np.linalg.norm(values))
if norm == 0.0:
raise ValueError("cannot normalize zero vector")
return values / norm
norms = np.linalg.norm(values, axis=1, keepdims=True)
if np.any(norms == 0.0):
raise ValueError("cannot normalize zero vector rows")
return values / norms
def _parse_orientation(value: object) -> int | None:
try:
orientation = int(value)
except (TypeError, ValueError):
return None
return orientation if 1 <= orientation <= 8 else None
def _iter_bmff_boxes(
payload: bytes,
*,
start: int = 0,
end: int | None = None,
) -> list[tuple[bytes, int, int, int]]:
if end is None:
end = len(payload)
boxes: list[tuple[bytes, int, int, int]] = []
cursor = start
while cursor + 8 <= end:
box_start = cursor
size_32 = int.from_bytes(payload[cursor : cursor + 4], byteorder="big")
box_type = payload[cursor + 4 : cursor + 8]
cursor += 8
if size_32 == 1:
if cursor + 8 > end:
break
box_size = int.from_bytes(payload[cursor : cursor + 8], byteorder="big")
cursor += 8
elif size_32 == 0:
box_size = end - box_start
else:
box_size = size_32
if box_type == b"uuid":
if cursor + 16 > end:
break
cursor += 16
header_size = cursor - box_start
if box_size < header_size:
break
box_end = box_start + box_size
if box_end > end:
break
boxes.append((box_type, box_start, cursor, box_end))
cursor = box_end
if box_size == 0:
break
return boxes
def _parse_pitm_item_id(meta_payload: bytes, pitm_box: tuple[bytes, int, int, int]) -> int | None:
_, _, payload_start, payload_end = pitm_box
if payload_start + 4 > payload_end:
return None
version = meta_payload[payload_start]
cursor = payload_start + 4
if version == 0:
if cursor + 2 > payload_end:
return None
return int.from_bytes(meta_payload[cursor : cursor + 2], byteorder="big")
if version == 1:
if cursor + 4 > payload_end:
return None
return int.from_bytes(meta_payload[cursor : cursor + 4], byteorder="big")
return None
def _parse_ipco_properties(
payload: bytes,
ipco_box: tuple[bytes, int, int, int],
) -> list[tuple[bytes, bytes]]:
_, _, payload_start, payload_end = ipco_box
properties: list[tuple[bytes, bytes]] = []
for prop_type, _, prop_payload_start, prop_payload_end in _iter_bmff_boxes(
payload, start=payload_start, end=payload_end
):
properties.append((prop_type, payload[prop_payload_start:prop_payload_end]))
return properties
def _parse_primary_item_ipma_associations(
payload: bytes,
ipma_box: tuple[bytes, int, int, int],
*,
primary_item_id: int,
) -> list[int]:
_, _, payload_start, payload_end = ipma_box
if payload_start + 8 > payload_end:
return []
version = payload[payload_start]
flags = int.from_bytes(payload[payload_start + 1 : payload_start + 4], byteorder="big")
cursor = payload_start + 4
entry_count = int.from_bytes(payload[cursor : cursor + 4], byteorder="big")
cursor += 4
use_extended_property_index = bool(flags & 0x1)
associations: list[int] = []
for _ in range(entry_count):
if version == 0:
if cursor + 2 > payload_end:
break
item_id = int.from_bytes(payload[cursor : cursor + 2], byteorder="big")
cursor += 2
elif version == 1:
if cursor + 4 > payload_end:
break
item_id = int.from_bytes(payload[cursor : cursor + 4], byteorder="big")
cursor += 4
else:
break
if cursor + 1 > payload_end:
break
association_count = payload[cursor]
cursor += 1
for _ in range(association_count):
if use_extended_property_index:
if cursor + 2 > payload_end:
break
value = int.from_bytes(payload[cursor : cursor + 2], byteorder="big")
cursor += 2
property_index = value & 0x7FFF
else:
if cursor + 1 > payload_end:
break
value = payload[cursor]
cursor += 1
property_index = value & 0x7F
if item_id == primary_item_id and property_index > 0:
associations.append(property_index)
return associations
def _heif_primary_item_has_orientation_transform(source_path: Path) -> bool:
try:
payload = source_path.read_bytes()
except Exception:
return False
top_level = _iter_bmff_boxes(payload)
meta_box = next((box for box in top_level if box[0] == b"meta"), None)
if meta_box is None:
return False
_, _, meta_payload_start, meta_payload_end = meta_box
if meta_payload_start + 4 > meta_payload_end:
return False
# Skip full box header (version + flags).
meta_children_start = meta_payload_start + 4
meta_children = _iter_bmff_boxes(payload, start=meta_children_start, end=meta_payload_end)
pitm_box = next((box for box in meta_children if box[0] == b"pitm"), None)
if pitm_box is None:
return False
primary_item_id = _parse_pitm_item_id(payload, pitm_box)
if primary_item_id is None:
return False
iprp_box = next((box for box in meta_children if box[0] == b"iprp"), None)
if iprp_box is None:
return False
_, _, iprp_payload_start, iprp_payload_end = iprp_box
iprp_children = _iter_bmff_boxes(payload, start=iprp_payload_start, end=iprp_payload_end)
ipco_properties: list[tuple[bytes, bytes]] = []
for ipco_box in [box for box in iprp_children if box[0] == b"ipco"]:
ipco_properties.extend(_parse_ipco_properties(payload, ipco_box))
if not ipco_properties:
return False
associated_property_indexes: list[int] = []
for ipma_box in [box for box in iprp_children if box[0] == b"ipma"]:
associated_property_indexes.extend(
_parse_primary_item_ipma_associations(
payload,
ipma_box,
primary_item_id=primary_item_id,
)
)
if not associated_property_indexes:
return False
for property_index in associated_property_indexes:
if property_index > len(ipco_properties):
continue
property_type, property_payload = ipco_properties[property_index - 1]
if property_type == b"imir":
return True
if property_type == b"irot" and property_payload:
if (property_payload[0] & 0x03) != 0:
return True
return False
def _correct_image_orientation(image: Image.Image, *, source_path: Path | None = None) -> Image.Image:
exif_orientation = _parse_orientation(image.getexif().get(EXIF_ORIENTATION_TAG, 1)) or 1
# pillow-heif can normalize EXIF orientation to 1 while preserving the original
# value in `image.info["original_orientation"]`.
original_orientation = _parse_orientation(image.info.get("original_orientation"))
corrected = ImageOps.exif_transpose(image)
if exif_orientation != 1:
return corrected
manual_method = ORIENTATION_TRANSPOSE_METHODS.get(original_orientation)
if manual_method is None:
return corrected
if source_path is not None and source_path.suffix.lower() in HEIF_EXTENSIONS:
# Some HEIF images carry irot/imir primary-item transforms. In those
# cases, applying original_orientation from pillow-heif can double-rotate.
if _heif_primary_item_has_orientation_transform(source_path):
return corrected
return corrected.transpose(manual_method)
def decode_image_rgb(source_path: Path) -> np.ndarray:
source_path = source_path.expanduser().resolve()
try:
with Image.open(source_path) as image:
image = _correct_image_orientation(image, source_path=source_path)
image = image.convert("RGB")
pixels = np.array(image, dtype=np.uint8, copy=True)
return np.ascontiguousarray(pixels)
except Exception as pillow_error:
fallback_enabled = os.environ.get(ALLOW_OPENCV_FALLBACK_ENV, "").strip().lower() in {
"1",
"true",
"yes",
"on",
}
if not fallback_enabled:
suffix = source_path.suffix.lower()
heif_note = (
" Install pillow-heif to decode HEIC/HEIF files."
if suffix in {".heic", ".heif"}
else ""
)
raise RuntimeError(
f"Failed to decode image '{source_path}' with Pillow ({pillow_error}). "
"OpenCV decode fallback is DISABLED by default because it may diverge from "
"Pillow EXIF-orientation behavior and can break parity trust."
f"{heif_note} To force fallback anyway, set {ALLOW_OPENCV_FALLBACK_ENV}=1."
) from pillow_error
warning_message = (
f"FALLING BACK to OpenCV decode for '{source_path}' after Pillow failure "
f"({pillow_error}). This can diverge from Pillow EXIF-orientation semantics and "
"may produce unreliable parity results."
)
warnings.warn(warning_message, RuntimeWarning, stacklevel=2)
print(f"[ml_parity][WARNING] {warning_message}", file=sys.stderr)
raw = np.fromfile(str(source_path), dtype=np.uint8)
decoded = cv2.imdecode(raw, cv2.IMREAD_COLOR)
if decoded is None:
suffix = source_path.suffix.lower()
heif_note = (
" Install pillow-heif to decode HEIC/HEIF files."
if suffix in {".heic", ".heif"}
else ""
)
raise RuntimeError(
f"Failed to decode image '{source_path}' with Pillow ({pillow_error})"
f" and OpenCV.{heif_note}"
) from pillow_error
rgb = cv2.cvtColor(decoded, cv2.COLOR_BGR2RGB)
return np.ascontiguousarray(rgb)