Skip to content

Production interior authoring and interactive Three.js delivery plugin for img2.

15.5 KB
Raw Blame
"""Explicit image recipes, not an image-to-material inference engine.

Schema 1: operations have id, op and output (workspace-relative PNG). Input is
{texture, channel}, {model, include}, {path}, or {reference: true}. A dye operation
has dye:{space:srgb255|linear,value:[r,g,b]}, bounds and optional strength,
exponent, normalizeMean, blur, crop. Roughness uses offset, scale, center
(zero|median), bounds. Crop uses crop:[left,top,right,bottom]. Woven uses outputs
{basecolor,normal,roughness}, resolution, yarns, tileMetres, seed and dye; optional
warpHeight, crossHeight, rippleHeight, yarnWidth, dyeBounds, noise, roughness,
roughnessScale all use physical/linear units as named. Inputs are never modified.
"""
import math
import os
from pathlib import Path
import tempfile

from .storage import confined, hashes, read_json, write_json

MAX_PIXELS = 16 * 1024 * 1024


def dependencies():
    try:
        import numpy as np
        from PIL import Image, ImageFilter
    except ImportError as exc:
        raise RuntimeError('Surface preparation needs optional requirements-surfaces.txt (NumPy and Pillow)') from exc
    return np, Image, ImageFilter


def number(value, name, lower=None, upper=None):
    if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
        raise ValueError(f'{name} must be a finite number')
    if (lower is not None and value < lower) or (upper is not None and value > upper):
        raise ValueError(f'{name} is outside its allowed range')
    return value


def bounds(value, name):
    if not isinstance(value, list) or len(value) != 2:
        raise ValueError(f'{name} requires [minimum, maximum]')
    lo, hi = [number(v, name) for v in value]
    if lo > hi:
        raise ValueError(f'{name} minimum exceeds maximum')
    return lo, hi


def linear(rgb, np):
    return np.where(rgb <= 0.04045, rgb / 12.92, ((rgb + 0.055) / 1.055) ** 2.4)


def srgb(rgb, np):
    return np.where(rgb <= 0.0031308, rgb * 12.92, 1.055 * np.maximum(rgb, 0) ** (1 / 2.4) - 0.055)


def dye_value(dye, np):
    if not isinstance(dye, dict) or dye.get('space') not in ('srgb255', 'linear'):
        raise ValueError('dye requires space srgb255 or linear')
    values = dye.get('value')
    if not isinstance(values, list) or len(values) != 3:
        raise ValueError('dye.value requires three color components')
    maximum = 255 if dye['space'] == 'srgb255' else 1
    color = np.asarray([number(v, 'dye', 0, maximum) for v in values], dtype=np.float32)
    return linear(color / 255, np) if maximum == 255 else color


def pixels(path, mode, Image):
    from PIL import ImageOps
    with Image.open(path) as image:
        if image.width * image.height > MAX_PIXELS:
            raise ValueError(f'Image exceeds {MAX_PIXELS} decoded pixels: {path}')
        return ImageOps.exif_transpose(image).convert(mode)


def statistics(image, np):
    array = np.asarray(image, dtype=np.float32) / 255
    if array.ndim == 2:
        array = array[..., None]
    return {'size': list(image.size), 'mode': image.mode, 'encoding': 'decoded uint8 / 255',
            'min': array.min(axis=(0, 1)).tolist(), 'max': array.max(axis=(0, 1)).tolist(),
            'mean': array.mean(axis=(0, 1)).tolist(), 'std': array.std(axis=(0, 1)).tolist()}


def crop_image(image, rectangle):
    if (not isinstance(rectangle, list) or len(rectangle) != 4 or
            any(type(v) is not int for v in rectangle)):
        raise ValueError('crop requires four integer pixel coordinates')
    left, top, right, bottom = rectangle
    if not 0 <= left < right <= image.width or not 0 <= top < bottom <= image.height:
        raise ValueError(f'Crop {rectangle} exceeds input image {image.size}; observe the actual image first')
    return image.crop(rectangle)


def transform(image, operation, np, ImageFilter):
    op = operation['op']
    if 'crop' in operation:
        image = crop_image(image, operation['crop'])
    if op == 'crop':
        if 'crop' not in operation:
            raise ValueError('crop operation requires an explicit crop rectangle')
        return np.asarray(image, dtype=np.float32) / 255
    if op == 'dye':
        blur = number(operation.get('blur', 0), 'blur', 0, 128)
        if blur:
            image = image.filter(ImageFilter.GaussianBlur(blur))
        scan = linear(np.asarray(image, dtype=np.float32) / 255, np)
        luminance = scan @ np.array([0.2126, 0.7152, 0.0722], dtype=np.float32)
        median = float(np.median(luminance))
        if median <= 0:
            raise ValueError('Cannot normalize a scan with zero median luminance')
        lo, hi = bounds(operation['bounds'], 'dye bounds')
        if lo < 0:
            raise ValueError('Dye modulation cannot be negative')
        ratio = np.clip(luminance / median, lo, hi)
        ratio = ratio ** number(operation.get('exponent', 1), 'exponent', 0.01, 10)
        ratio = 1 + number(operation.get('strength', 1), 'strength', 0, 1) * (ratio - 1)
        normalize = operation.get('normalizeMean', False)
        if type(normalize) is not bool:
            raise ValueError('normalizeMean must be boolean')
        if normalize:
            if float(ratio.mean()) <= 0:
                raise ValueError('Cannot normalize zero modulation')
            ratio /= ratio.mean()
        return np.clip(srgb(ratio[..., None] * dye_value(operation['dye'], np), np), 0, 1)
    if op == 'roughness':
        scan = np.asarray(image, dtype=np.float32) / 255
        center = operation.get('center', 'zero')
        if center not in ('zero', 'median'):
            raise ValueError('roughness center must be zero or median')
        lo, hi = bounds(operation.get('bounds', [0, 1]), 'roughness bounds')
        if lo < 0 or hi > 1:
            raise ValueError('Roughness bounds must lie within [0,1]')
        return np.clip(number(operation['offset'], 'offset') + number(operation['scale'], 'scale') *
                       (scan - (np.median(scan) if center == 'median' else 0)), lo, hi)
    raise ValueError(f'Unknown surface operation: {op}')


def woven(operation, np):
    resolution, yarns = operation['resolution'], operation['yarns']
    if type(resolution) is not int or not 4 <= resolution <= 4096:
        raise ValueError('Woven resolution must be an integer in [4,4096]')
    if type(yarns) is not int or not 2 <= yarns <= resolution or yarns % 2:
        raise ValueError('Woven yarns must be an even integer in [2,resolution] for periodic crossing parity')
    tile = number(operation['tileMetres'], 'tileMetres', 0.0001, 100)
    seed = operation['seed']
    if type(seed) is not int or seed < 0:
        raise ValueError('Woven seed must be a nonnegative integer')
    coordinate = (np.arange(resolution, dtype=np.float32) + 0.5) * yarns / resolution
    x, y = coordinate[None, :], coordinate[:, None]
    ix, iy = np.floor(x).astype(np.int32), np.floor(y).astype(np.int32)
    fx, fy = x % 1 - 0.5, y % 1 - 0.5
    width = number(operation.get('yarnWidth', 0.49), 'yarnWidth', 0.01, 0.5)
    warp = np.sqrt(np.maximum(1 - (fx / width) ** 2, 0))
    weft = np.sqrt(np.maximum(1 - (fy / width) ** 2, 0))
    over = ((ix + iy) % 2) == 0
    high = number(operation.get('warpHeight', 0.00055), 'warpHeight', 0, 1)
    low = number(operation.get('crossHeight', 0.00009), 'crossHeight', 0, 1)
    ripple = number(operation.get('rippleHeight', 0.000012), 'rippleHeight', 0, 1)
    height = np.where(over, high * warp + low * weft, high * weft + low * warp)
    height += ripple * np.sin(x * np.pi * 8) * np.sin(y * np.pi * 6)
    du = (np.roll(height, -1, axis=1) - np.roll(height, 1, axis=1)) * resolution / (2 * tile)
    dv = (np.roll(height, -1, axis=0) - np.roll(height, 1, axis=0)) * resolution / (2 * tile)
    # PNG rows point down; OpenGL tangent-space +V points up.
    normal = np.stack((-du, dv, np.ones_like(height)), axis=-1)
    normal /= np.linalg.norm(normal, axis=-1, keepdims=True)
    rng = np.random.default_rng(seed)
    lo, hi = bounds(operation.get('dyeBounds', [0.92, 1.08]), 'dyeBounds')
    if lo < 0:
        raise ValueError('Yarn dye modulation cannot be negative')
    yarn_dye = rng.uniform(lo, hi, (yarns, 2)).astype(np.float32)
    modulation = np.where(over, yarn_dye[ix, 0], yarn_dye[iy, 1])
    modulation += rng.normal(0, number(operation.get('noise', 0.007), 'noise', 0, 1),
                             (resolution, resolution)).astype(np.float32)
    color = np.clip(srgb(modulation[..., None] * dye_value(operation['dye'], np), np), 0, 1)
    roughness = np.clip(number(operation.get('roughness', 0.86), 'roughness', 0, 1) +
                        number(operation.get('roughnessScale', 0.05), 'roughnessScale') * (modulation - 1), 0, 1)
    return {'basecolor': color, 'normal': normal * 0.5 + 0.5, 'roughness': roughness}, {
        'tileMetres': tile, 'yarnPitchMetres': tile / yarns,
        'normalConvention': 'OpenGL tangent +Y; periodic central derivatives of physical height',
        'maximumNormalLengthError': float(np.max(np.abs(np.linalg.norm(normal, axis=-1) - 1)))}


def reference_path(workspace, recipe):
    spec = confined(workspace, recipe.get('spec', 'scene-spec.json'))
    return read_json(spec)['source'] if spec.exists() else 'public/assets/reference.png'


def resolve_input(workspace, reference, manifest, value):
    if not isinstance(value, dict):
        raise ValueError('input must be a path, reference, texture/channel or model/include object')
    if set(value) == {'path'}:
        relative = value['path']
    elif value == {'reference': True}:
        relative = reference
    elif set(value) == {'texture', 'channel'}:
        if value['channel'] not in ('color', 'normal', 'roughness'):
            raise ValueError('Texture channel must be color, normal or roughness')
        relative = manifest['textures'][value['texture']][value['channel']]
    elif set(value) == {'model', 'include'}:
        relative = manifest['modelIncludes'][value['model']][value['include']]
    else:
        raise ValueError(f'Ambiguous or unsupported surface input: {value}')
    path = confined(workspace, relative)
    if not path.is_file():
        raise ValueError(f'Missing surface input: {relative}')
    if 'texture' in value or 'model' in value:
        receipt = next((entry for entry in manifest['downloads'] if entry['path'] == relative), None)
        if receipt is None or hashes(path)[2] != receipt['sha256']:
            raise ValueError(f'Resource integrity mismatch: {relative}; fetch resources again')
    return relative, path


def save_image(workspace, relative, array, np, Image):
    if not np.isfinite(array).all():
        raise ValueError('Non-finite generated pixels')
    path = confined(workspace, relative)
    path.parent.mkdir(parents=True, exist_ok=True)
    image = Image.fromarray(np.rint(np.clip(array, 0, 1) * 255).astype(np.uint8))
    fd, temporary = tempfile.mkstemp(prefix='.surface-', suffix='.png', dir=path.parent)
    os.close(fd)
    try:
        image.save(temporary, format='PNG', optimize=True)
        os.replace(temporary, confined(workspace, relative))
    finally:
        if os.path.exists(temporary):
            os.unlink(temporary)
    decoded = pixels(path, image.mode, Image)
    return {'path': relative, 'sha256': hashes(path)[2], 'pixels': statistics(decoded, np)}


def prepare(workspace, recipe_path):
    np, Image, ImageFilter = dependencies()
    workspace = Path(workspace).resolve()
    recipe_file = confined(workspace, recipe_path)
    recipe = read_json(recipe_file)
    if not isinstance(recipe, dict) or type(recipe.get('schemaVersion')) is not int or recipe['schemaVersion'] != 1:
        raise ValueError('Surface recipe must have schemaVersion 1')
    operations = recipe.get('operations')
    if not isinstance(operations, list) or not operations or len(operations) > 100:
        raise ValueError('Surface recipe needs 1–100 operations')
    reference = reference_path(workspace, recipe)
    if 'referenceSha256' in recipe:
        if hashes(confined(workspace, reference))[2] != recipe['referenceSha256']:
            raise ValueError('This recipe is bound to a different reference image; author a new recipe instead of reusing its crops')
    manifest_relative = recipe.get('resourceManifest', '.img2/artifacts/interior/resources.json')
    manifest_file = confined(workspace, manifest_relative)
    needs_resources = any(isinstance(op, dict) and isinstance(op.get('input'), dict) and
                          ('texture' in op['input'] or 'model' in op['input']) for op in operations)
    manifest = read_json(manifest_file) if needs_resources else {}
    protected = {recipe_file, manifest_file, confined(workspace, reference),
                 confined(workspace, recipe.get('spec', 'scene-spec.json'))}
    planned, output_paths, ids = [], set(), set()
    for operation in operations:
        if not isinstance(operation, dict) or operation.get('op') not in ('dye', 'roughness', 'crop', 'woven'):
            raise ValueError('Each operation must be dye, roughness, crop or woven')
        identifier = operation.get('id')
        if not isinstance(identifier, str) or not identifier.strip() or identifier in ids:
            raise ValueError('Operations need unique nonempty ids')
        ids.add(identifier)
        source = None
        if operation['op'] == 'woven':
            outputs = operation.get('outputs')
            if not isinstance(outputs, dict) or set(outputs) != {'basecolor', 'normal', 'roughness'}:
                raise ValueError('Woven outputs must name basecolor, normal and roughness')
        else:
            source = resolve_input(workspace, reference, manifest, operation.get('input'))
            protected.add(source[1])
            outputs = {'image': operation.get('output')}
        for relative in outputs.values():
            path = confined(workspace, relative)
            if path.suffix.lower() != '.png' or not relative.startswith('public/assets/interior/'):
                raise ValueError('Surface outputs must be PNGs under public/assets/interior/')
            if path in output_paths:
                raise ValueError(f'Duplicate surface output: {relative}')
            output_paths.add(path)
        planned.append((operation, source, outputs))
    if output_paths & protected:
        raise ValueError('Surface outputs must not overwrite any recipe, source or resource input')
    records = []
    for operation, source, outputs in planned:
        record = {'id': operation['id'], 'operation': operation, 'inputs': [], 'outputs': {}}
        if operation['op'] == 'woven':
            arrays, metric = woven(operation, np)
            record.update(metric)
        else:
            relative, path = source
            image = pixels(path, 'L' if operation['op'] == 'roughness' else 'RGB', Image)
            record['inputs'].append({'path': relative, 'sha256': hashes(path)[2], 'pixels': statistics(image, np)})
            arrays = {'image': transform(image, operation, np, ImageFilter)}
        for channel, array in arrays.items():
            record['outputs'][channel] = save_image(workspace, outputs[channel], array, np, Image)
        records.append(record)
    receipt = {'schemaVersion': 1, 'recipe': recipe_path, 'recipeSha256': hashes(recipe_file)[2],
               'operations': records, 'limits': recipe.get('limits', []),
               'tools': {'numpy': np.__version__, 'Pillow': Image.__version__},
               'interpretation': 'Explicit authored transformations, not recovered physical reflectance or visual acceptance'}
    if needs_resources:
        receipt['resourceManifest'] = manifest_relative
        receipt['resourceManifestSha256'] = hashes(manifest_file)[2]
    write_json(workspace, '.img2/artifacts/interior/surfaces.json', receipt)
    return receipt