Free open-source image editor for the browser
Add HEIC image decoding
Rodrigobc7f653a914b
diff --git a/src/app/editor.tsx b/src/app/editor.tsxindex 8a8c454..378dbd9 100644--- a/src/app/editor.tsx+++ b/src/app/editor.tsx@@ -8,23 +8,31 @@ import usePanZoom, { type View } from "@/hooks/use-pan-zoom";import decode, { type Target } from "@/lib/decode";import shader from "./editor.wgsl";+type Decoding = { image?: Target; error?: string };+/** Decodes the file into the GPU, disposing it on cleanup. */function useDecode(file: File) {const gpu = useGpu();- const [image, setImage] = useState<Target>();+ const [state, setState] = useState<Decoding>({});useEffect(() => {let cancelled = false;const decoding = decode(gpu, file);- decoding.then((image) => !cancelled && setImage(image));+ decoding.then(+ (image) => !cancelled && setState({ image }),+ (error: Error) => !cancelled && setState({ error: error.message }),+ );return () => {cancelled = true;- decoding.then((image) => image.color.dispose());- setImage(undefined);+ decoding.then(+ (image) => image.color.dispose(),+ () => {},+ );+ setState({});};}, [gpu, file]);- return image;+ return state;}type ImageProps = { image: Target; view: View };@@ -57,7 +65,7 @@ function Image({ image, view }: ImageProps) {type EditorProps = { file: File };export default function Editor({ file }: EditorProps) {- const image = useDecode(file);+ const { image, error } = useDecode(file);const { ref, view, handlers } = usePanZoom(image?.size);return (@@ -71,6 +79,10 @@ export default function Editor({ file }: EditorProps) {<Canvas className="size-full min-h-0"><Image image={image} view={view} /></Canvas>+ ) : error ? (+ <p className="max-w-md text-center text-neutral-400">+ Couldn't open {file.name}: {error}+ </p>) : (<Spinner />)}diff --git a/src/hooks/use-pan-zoom.ts b/src/hooks/use-pan-zoom.tsindex 6b1dba6..f1fe5bc 100644--- a/src/hooks/use-pan-zoom.ts+++ b/src/hooks/use-pan-zoom.ts@@ -71,7 +71,8 @@ export default function usePanZoom(content?: Size) {if (!element) {return;}- update((view) => view);+ const observer = new ResizeObserver(() => update((view) => view));+ observer.observe(element);const wheel = (event: WheelEvent) => {event.preventDefault();const { left, top, width, height } = element.getBoundingClientRect();@@ -89,7 +90,10 @@ export default function usePanZoom(content?: Size) {);};element.addEventListener("wheel", wheel, { passive: false });- return () => element.removeEventListener("wheel", wheel);+ return () => {+ observer.disconnect();+ element.removeEventListener("wheel", wheel);+ };}, [update]);const handlers = {diff --git a/src/lib/color.wgsl b/src/lib/color.wgslindex 8dd1edd..e8fd090 100644--- a/src/lib/color.wgsl+++ b/src/lib/color.wgsl@@ -1,12 +1,18 @@import { linearToSrgb3 } from "@vgpu/wgsl-std/color";-// The working space is linear Rec.2020; these convert its primaries to and from linear sRGB (column-major).+// The working space is linear Rec.2020; these convert its primaries to and from linear sRGB and Display P3 (column-major).export const srgbToRec2020 = mat3x3f(0.6274, 0.0691, 0.0164,0.3293, 0.9195, 0.0880,0.0433, 0.0114, 0.8956,);+export const p3ToRec2020 = mat3x3f(+ 0.7538, 0.0457, -0.0012,+ 0.1986, 0.9418, 0.0176,+ 0.0476, 0.0125, 0.9836,+);+export const rec2020ToSrgb = mat3x3f(1.6605, -0.1246, -0.0182,-0.5876, 1.1329, -0.1006,diff --git a/src/lib/decode/heic.ts b/src/lib/decode/heic.tsnew file mode 100644index 0000000..b912eb0--- /dev/null+++ b/src/lib/decode/heic.ts@@ -0,0 +1,262 @@+/**+ * HEIC through the browser's HEVC decoder (WebCodecs). Reads just enough HEIF to find the primary+ * image — one hvc1 item or a grid of hvc1 tiles — and composes the decoded frames on a canvas.+ */++type Box = { type: string; start: number; end: number };++class Reader {+ at = 0;+ readonly view: DataView;++ constructor(readonly bytes: Uint8Array) {+ this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);+ }++ /** Big-endian unsigned integer of `size` bytes (0, 1, 2, 4 or 8). */+ uint(size: number) {+ const at = this.at;+ this.at += size;+ switch (size) {+ case 1:+ return this.view.getUint8(at);+ case 2:+ return this.view.getUint16(at);+ case 4:+ return this.view.getUint32(at);+ case 8:+ return Number(this.view.getBigUint64(at));+ default:+ return 0;+ }+ }++ fourcc() {+ const start = this.at;+ this.at += 4;+ return String.fromCharCode(...this.bytes.subarray(start, this.at));+ }++ /** Positions at a full box's content and returns its version byte. */+ open(box: Box) {+ this.at = box.start;+ return this.uint(4) >>> 24;+ }++ /** Boxes in [start, end), each spanning its content. */+ boxes(start: number, end: number) {+ const list: Box[] = [];+ for (this.at = start; this.at + 8 <= end; ) {+ const head = this.at;+ const size = this.uint(4);+ const type = this.fourcc();+ const length = size === 1 ? this.uint(8) : size === 0 ? end - head : size;+ list.push({ type, start: this.at, end: head + length });+ this.at = head + length;+ }+ return list;+ }++ content(box: Box) {+ return this.bytes.subarray(box.start, box.end);+ }+}++function find(boxes: Box[], type: string) {+ const box = boxes.find((b) => b.type === type);+ if (!box) {+ throw new Error(`HEIF: missing ${type} box`);+ }+ return box;+}++/** iinf: item id → item type. */+function itemTypes(reader: Reader, iinf: Box) {+ const types = new Map<number, string>();+ const count = reader.uint(reader.open(iinf) === 0 ? 2 : 4);+ for (const infe of reader.boxes(reader.at, iinf.end).slice(0, count)) {+ const id = reader.uint(reader.open(infe) === 2 ? 2 : 4);+ reader.at += 2;+ types.set(id, reader.fourcc());+ }+ return types;+}++/** iloc: item id → its bytes, joining extents from the file or the idat box. */+function itemData(reader: Reader, iloc: Box, idat?: Box) {+ const data = new Map<number, Uint8Array>();+ const version = reader.open(iloc);+ const sizes = reader.uint(2);+ const [offsetSize, lengthSize, baseSize] = [+ sizes >> 12,+ (sizes >> 8) & 15,+ (sizes >> 4) & 15,+ ];+ const indexSize = version === 0 ? 0 : sizes & 15;+ const idSize = version < 2 ? 2 : 4;+ for (let count = reader.uint(idSize); count > 0; count--) {+ const id = reader.uint(idSize);+ const method = version === 0 ? 0 : reader.uint(2) & 15;+ reader.at += 2;+ const base =+ reader.uint(baseSize) + (method === 1 ? (idat?.start ?? 0) : 0);+ const extents = Array.from({ length: reader.uint(2) }, () => {+ reader.at += indexSize;+ const start = base + reader.uint(offsetSize);+ return reader.bytes.subarray(start, start + reader.uint(lengthSize));+ });+ const joined = new Uint8Array(+ extents.reduce((n, e) => n + e.byteLength, 0),+ );+ let at = 0;+ for (const extent of extents) {+ joined.set(extent, at);+ at += extent.byteLength;+ }+ data.set(id, joined);+ }+ return data;+}++/** iprp: item id → its property boxes (ipco entries, associated through ipma). */+function itemProperties(reader: Reader, iprp: Box) {+ const children = reader.boxes(iprp.start, iprp.end);+ const ipco = find(children, "ipco");+ const properties = reader.boxes(ipco.start, ipco.end);+ const associations = new Map<number, Box[]>();+ for (const ipma of children.filter((b) => b.type === "ipma")) {+ reader.at = ipma.start;+ const versionFlags = reader.uint(4);+ const wide = versionFlags & 1;+ for (let count = reader.uint(4); count > 0; count--) {+ const id = reader.uint(versionFlags >>> 24 === 0 ? 2 : 4);+ const boxes = Array.from({ length: reader.uint(1) }, () => {+ const index = reader.uint(wide ? 2 : 1) & (wide ? 0x7fff : 0x7f);+ return properties[index - 1];+ });+ associations.set(+ id,+ boxes.filter((b) => b !== undefined),+ );+ }+ }+ return associations;+}++/** iref dimg: derived item id → source item ids, in order. */+function itemSources(reader: Reader, iref?: Box) {+ const sources = new Map<number, number[]>();+ if (iref) {+ const idSize = reader.open(iref) === 0 ? 2 : 4;+ for (const ref of reader.boxes(reader.at, iref.end)) {+ reader.at = ref.start;+ const from = reader.uint(idSize);+ const to = Array.from({ length: reader.uint(2) }, () =>+ reader.uint(idSize),+ );+ if (ref.type === "dimg") {+ sources.set(from, to);+ }+ }+ }+ return sources;+}++/** WebCodecs codec string for an HEVC decoder configuration record (hvcC). */+function codecOf(hvcC: Uint8Array) {+ const profile = hvcC[1] ?? 0;+ const view = new DataView(hvcC.buffer, hvcC.byteOffset);+ let compatibility = 0;+ for (let i = 0; i < 32; i++) {+ compatibility =+ ((compatibility << 1) | ((view.getUint32(2) >>> i) & 1)) >>> 0;+ }+ const constraints = Array.from(hvcC.subarray(6, 12), (b) =>+ b.toString(16).padStart(2, "0"),+ );+ while (constraints.at(-1) === "00") {+ constraints.pop();+ }+ const space = ["", "A", "B", "C"][profile >> 6];+ const tier = profile & 0x20 ? "H" : "L";+ return [+ `hvc1.${space}${profile & 31}`,+ compatibility.toString(16),+ `${tier}${hvcC[12]}`,+ ...constraints,+ ].join(".");+}++/** ispe: [width, height]. */+function sizeOf(ispe: Uint8Array) {+ const view = new DataView(ispe.buffer, ispe.byteOffset);+ return [view.getUint32(4), view.getUint32(8)] as const;+}++export default async function decodeHeic(file: Blob) {+ const reader = new Reader(new Uint8Array(await file.arrayBuffer()));+ const meta = find(reader.boxes(0, reader.bytes.byteLength), "meta");+ const boxes = reader.boxes(meta.start + 4, meta.end);+ const primary = reader.uint(reader.open(find(boxes, "pitm")) === 0 ? 2 : 4);+ const types = itemTypes(reader, find(boxes, "iinf"));+ const data = itemData(+ reader,+ find(boxes, "iloc"),+ boxes.find((b) => b.type === "idat"),+ );+ const properties = itemProperties(reader, find(boxes, "iprp"));+ const sources = itemSources(+ reader,+ boxes.find((b) => b.type === "iref"),+ );+ const property = (id: number, type: string) => {+ const box = properties.get(id)?.find((b) => b.type === type);+ return box && reader.content(box);+ };++ const grid = types.get(primary) === "grid" ? data.get(primary) : undefined;+ const items = grid ? (sources.get(primary) ?? []) : [primary];+ const hvcC = property(items[0] ?? primary, "hvcC");+ const ispe = property(primary, "ispe");+ if (!hvcC || !ispe) {+ throw new Error("HEIF: primary item is not an HEVC image");+ }+ const [width, height] = sizeOf(ispe);++ const tiles: VideoFrame[] = [];+ await new Promise<void>((resolve, reject) => {+ const decoder = new VideoDecoder({+ output: (frame) => {+ tiles[frame.timestamp] = frame;+ },+ error: (error) => {+ for (const frame of tiles) {+ frame?.close();+ }+ reject(error);+ },+ });+ decoder.configure({ codec: codecOf(hvcC), description: hvcC });+ items.forEach((id, timestamp) => {+ decoder.decode(+ new EncodedVideoChunk({+ type: "key",+ timestamp,+ data: data.get(id) ?? new Uint8Array(),+ }),+ );+ });+ decoder.flush().then(() => {+ decoder.close();+ resolve();+ }, reject);+ });++ return {+ width,+ height,+ tiles,+ columns: grid ? (grid[3] ?? 0) + 1 : 1,+ rotation: (property(primary, "irot")?.[0] ?? 0) & 3,+ };+}diff --git a/src/lib/decode/index.ts b/src/lib/decode/index.tsindex dff964b..95364ca 100644--- a/src/lib/decode/index.ts+++ b/src/lib/decode/index.ts@@ -1,11 +1,15 @@import type { Gpu, Target } from "vgpu";+import decodeHeic from "./heic";import linearize from "./linearize";import decodeSvg from "./svg";import type { Decoder, Pixels } from "./types";export type { Target };-/** Decoder backed by a worker module: post the file, receive transferred pixels or an error. */+/**+ * Decoder backed by a worker module: post the file, receive transferred pixels or an error.+ * Unused since HEIC moved to WebCodecs; kept for CPU-heavy decoders to come (RAW).+ */export function workerDecoder(load: () => Promise<{ default: new () => Worker }>,) {@@ -33,6 +37,7 @@ type Format = {const native = async () => createImageBitmap;const svg = async () => decodeSvg;+const heic = async () => decodeHeic;const formats: Format[] = [{ types: ["image/png"], extensions: ["png"], load: native },@@ -42,6 +47,11 @@ const formats: Format[] = [{ types: ["image/avif"], extensions: ["avif"], load: native },{ types: ["image/bmp"], extensions: ["bmp"], load: native },{ types: ["image/svg+xml"], extensions: ["svg"], load: svg },+ {+ types: ["image/heic", "image/heif"],+ extensions: ["heic", "heif"],+ load: heic,+ },];/** Matches by MIME type, then by extension for files the OS doesn't type. */diff --git a/src/lib/decode/linearize.ts b/src/lib/decode/linearize.tsindex e18fda3..197c420 100644--- a/src/lib/decode/linearize.ts+++ b/src/lib/decode/linearize.ts@@ -2,39 +2,83 @@ import { effect, frame, type Gpu, target } from "vgpu";import shader from "./linearize.wgsl";import type { Decoded } from "./types";-type Size = readonly [number, number];+type Size = [number, number];-/** Stages the decoded bytes in a texture whose format decodes them to linear when read. */-function stage(gpu: Gpu, decoded: Decoded, size: Size) {- const texture = gpu.device.createTexture({+/** Display P3 primaries, which TypeScript's VideoColorPrimaries doesn't list yet. */+const isP3 = (frame?: VideoFrame) =>+ (frame?.colorSpace.primaries as string) === "smpte432";++/** A texture whose format decodes sRGB-encoded bytes to linear when read. */+function staging(gpu: Gpu, size: Size) {+ return gpu.device.createTexture({size,format: "rgba8unorm-srgb",usage: ["texture_binding", "copy_dst", "render_attachment"],});+}++/** Uploads the decoded image into a staging texture, tiles laid out on their grid. */+function stage(gpu: Gpu, decoded: Decoded) {+ const queue = gpu.gpu.queue;+ const size: Size = [decoded.width, decoded.height];if (decoded instanceof ImageBitmap) {- gpu.gpu.queue.copyExternalImageToTexture(+ const texture = staging(gpu, size);+ queue.copyExternalImageToTexture({ source: decoded },{ texture: texture.gpu },size,);decoded.close();- } else {- gpu.gpu.queue.writeTexture(- { texture: texture.gpu },- decoded.data,- { bytesPerRow: decoded.width * 4 },- size,- );+ return texture;+ }+ if ("tiles" in decoded) {+ const { tiles, columns } = decoded;+ const tile: Size = [+ tiles[0]?.displayWidth ?? 0,+ tiles[0]?.displayHeight ?? 0,+ ];+ const rows = Math.ceil(tiles.length / columns);+ const texture = staging(gpu, [columns * tile[0], rows * tile[1]]);+ tiles.forEach((frame, i) => {+ const origin = [+ (i % columns) * tile[0],+ Math.floor(i / columns) * tile[1],+ ];+ const colorSpace = isP3(frame) ? "display-p3" : "srgb";+ queue.copyExternalImageToTexture(+ { source: frame },+ { texture: texture.gpu, origin, colorSpace },+ tile,+ );+ frame.close();+ });+ return texture;}+ const texture = staging(gpu, size);+ queue.writeTexture(+ { texture: texture.gpu },+ decoded.data,+ { bytesPerRow: decoded.width * 4 },+ size,+ );return texture;}/** GPU leg for sRGB-encoded decoders: renders the decoded image into a linear Rec.2020 rgba16float target. */export default function linearize(gpu: Gpu, decoded: Decoded) {+ const frames = "tiles" in decoded ? decoded : undefined;+ const rotation = frames?.rotation ?? 0;+ const p3 = Number(isP3(frames?.tiles[0]));const size: Size = [decoded.width, decoded.height];- const source = stage(gpu, decoded, size);- const image = target(gpu, { size, format: "rgba16float" });- frame(gpu, (f) => f.pass(image, effect(gpu, shader, { set: { source } })));+ const source = stage(gpu, decoded);+ const image = target(gpu, {+ size: rotation % 2 ? [size[1], size[0]] : size,+ format: "rgba16float",+ });+ const params = { size, rotation, p3 };+ frame(gpu, (f) =>+ f.pass(image, effect(gpu, shader, { set: { source, params } })),+ );source.dispose();return image;}diff --git a/src/lib/decode/linearize.wgsl b/src/lib/decode/linearize.wgslindex 6adac06..7bb7ebc 100644--- a/src/lib/decode/linearize.wgsl+++ b/src/lib/decode/linearize.wgsl@@ -1,8 +1,23 @@-import { srgbToRec2020 } from "../color.wgsl";+import { p3ToRec2020, srgbToRec2020 } from "../color.wgsl";-@group(0) @binding(0) var source: texture_2d<f32>;+struct Params {+ size: vec2f,+ rotation: u32,+ p3: u32,+}+@group(0) @binding(0) var<uniform> params: Params;+@group(0) @binding(1) var source: texture_2d<f32>;-@fragment fn fs_main(@builtin(position) position: vec4f) -> @location(0) vec4f {- let texel = textureLoad(source, vec2i(position.xy), 0);- return vec4f(srgbToRec2020 * texel.rgb, texel.a);+@fragment fn fs_main(@location(0) uv: vec2f) -> @location(0) vec4f {+ // Undo `rotation` counter-clockwise quarter turns: where this output pixel sits in the source.+ var p = uv;+ switch (params.rotation) {+ case 1u: { p = vec2f(1.0 - uv.y, uv.x); }+ case 2u: { p = 1.0 - uv; }+ case 3u: { p = vec2f(uv.y, 1.0 - uv.x); }+ default: {}+ }+ let texel = textureLoad(source, vec2i(p * params.size), 0);+ let color = select(srgbToRec2020 * texel.rgb, p3ToRec2020 * texel.rgb, params.p3 == 1u);+ return vec4f(color, texel.a);}diff --git a/src/lib/decode/types.ts b/src/lib/decode/types.tsindex 823c0da..c693006 100644--- a/src/lib/decode/types.ts+++ b/src/lib/decode/types.ts@@ -4,5 +4,13 @@ export type Pixels = {height: number;data: Uint8ClampedArray<ArrayBuffer>;};-export type Decoded = ImageBitmap | Pixels;+/** Video frames tiled on a grid `columns` wide, cropped to width × height, then rotated counter-clockwise by `rotation` quarter turns. */+export type Frames = {+ width: number;+ height: number;+ tiles: VideoFrame[];+ columns: number;+ rotation: number;+};+export type Decoded = ImageBitmap | Pixels | Frames;export type Decoder = (file: Blob) => Promise<Decoded>;