Skip to content

Free open-source image editor for the browser

Add HEIC image decoding

Rodrigobc7f653a914b
diff --git a/src/app/editor.tsx b/src/app/editor.tsx
index 8a8c454..378dbd9 100644
--- a/src/app/editor.tsx
+++ b/src/app/editor.tsx
@@ -8,23 +8,31 @@ import usePanZoom, { type View } from "@/hooks/use-pan-zoom";
import decode, { type Target } from "@/lib/decode";
import shader from "./editor.wgsl";
+type Decoding = { image?: Target; error?: string };
+
/** Decodes the file into the GPU, disposing it on cleanup. */
function useDecode(file: File) {
const gpu = useGpu();
- const [image, setImage] = useState<Target>();
+ const [state, setState] = useState<Decoding>({});
useEffect(() => {
let cancelled = false;
const decoding = decode(gpu, file);
- decoding.then((image) => !cancelled && setImage(image));
+ decoding.then(
+ (image) => !cancelled && setState({ image }),
+ (error: Error) => !cancelled && setState({ error: error.message }),
+ );
return () => {
cancelled = true;
- decoding.then((image) => image.color.dispose());
- setImage(undefined);
+ decoding.then(
+ (image) => image.color.dispose(),
+ () => {},
+ );
+ setState({});
};
}, [gpu, file]);
- return image;
+ return state;
}
type ImageProps = { image: Target; view: View };
@@ -57,7 +65,7 @@ function Image({ image, view }: ImageProps) {
type EditorProps = { file: File };
export default function Editor({ file }: EditorProps) {
- const image = useDecode(file);
+ const { image, error } = useDecode(file);
const { ref, view, handlers } = usePanZoom(image?.size);
return (
@@ -71,6 +79,10 @@ export default function Editor({ file }: EditorProps) {
<Canvas className="size-full min-h-0">
<Image image={image} view={view} />
</Canvas>
+ ) : error ? (
+ <p className="max-w-md text-center text-neutral-400">
+ Couldn't open {file.name}: {error}
+ </p>
) : (
<Spinner />
)}
diff --git a/src/hooks/use-pan-zoom.ts b/src/hooks/use-pan-zoom.ts
index 6b1dba6..f1fe5bc 100644
--- a/src/hooks/use-pan-zoom.ts
+++ b/src/hooks/use-pan-zoom.ts
@@ -71,7 +71,8 @@ export default function usePanZoom(content?: Size) {
if (!element) {
return;
}
- update((view) => view);
+ const observer = new ResizeObserver(() => update((view) => view));
+ observer.observe(element);
const wheel = (event: WheelEvent) => {
event.preventDefault();
const { left, top, width, height } = element.getBoundingClientRect();
@@ -89,7 +90,10 @@ export default function usePanZoom(content?: Size) {
);
};
element.addEventListener("wheel", wheel, { passive: false });
- return () => element.removeEventListener("wheel", wheel);
+ return () => {
+ observer.disconnect();
+ element.removeEventListener("wheel", wheel);
+ };
}, [update]);
const handlers = {
diff --git a/src/lib/color.wgsl b/src/lib/color.wgsl
index 8dd1edd..e8fd090 100644
--- a/src/lib/color.wgsl
+++ b/src/lib/color.wgsl
@@ -1,12 +1,18 @@
import { linearToSrgb3 } from "@vgpu/wgsl-std/color";
-// The working space is linear Rec.2020; these convert its primaries to and from linear sRGB (column-major).
+// The working space is linear Rec.2020; these convert its primaries to and from linear sRGB and Display P3 (column-major).
export const srgbToRec2020 = mat3x3f(
0.6274, 0.0691, 0.0164,
0.3293, 0.9195, 0.0880,
0.0433, 0.0114, 0.8956,
);
+export const p3ToRec2020 = mat3x3f(
+ 0.7538, 0.0457, -0.0012,
+ 0.1986, 0.9418, 0.0176,
+ 0.0476, 0.0125, 0.9836,
+);
+
export const rec2020ToSrgb = mat3x3f(
1.6605, -0.1246, -0.0182,
-0.5876, 1.1329, -0.1006,
diff --git a/src/lib/decode/heic.ts b/src/lib/decode/heic.ts
new file mode 100644
index 0000000..b912eb0
--- /dev/null
+++ b/src/lib/decode/heic.ts
@@ -0,0 +1,262 @@
+/**
+ * HEIC through the browser's HEVC decoder (WebCodecs). Reads just enough HEIF to find the primary
+ * image — one hvc1 item or a grid of hvc1 tiles — and composes the decoded frames on a canvas.
+ */
+
+type Box = { type: string; start: number; end: number };
+
+class Reader {
+ at = 0;
+ readonly view: DataView;
+
+ constructor(readonly bytes: Uint8Array) {
+ this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+ }
+
+ /** Big-endian unsigned integer of `size` bytes (0, 1, 2, 4 or 8). */
+ uint(size: number) {
+ const at = this.at;
+ this.at += size;
+ switch (size) {
+ case 1:
+ return this.view.getUint8(at);
+ case 2:
+ return this.view.getUint16(at);
+ case 4:
+ return this.view.getUint32(at);
+ case 8:
+ return Number(this.view.getBigUint64(at));
+ default:
+ return 0;
+ }
+ }
+
+ fourcc() {
+ const start = this.at;
+ this.at += 4;
+ return String.fromCharCode(...this.bytes.subarray(start, this.at));
+ }
+
+ /** Positions at a full box's content and returns its version byte. */
+ open(box: Box) {
+ this.at = box.start;
+ return this.uint(4) >>> 24;
+ }
+
+ /** Boxes in [start, end), each spanning its content. */
+ boxes(start: number, end: number) {
+ const list: Box[] = [];
+ for (this.at = start; this.at + 8 <= end; ) {
+ const head = this.at;
+ const size = this.uint(4);
+ const type = this.fourcc();
+ const length = size === 1 ? this.uint(8) : size === 0 ? end - head : size;
+ list.push({ type, start: this.at, end: head + length });
+ this.at = head + length;
+ }
+ return list;
+ }
+
+ content(box: Box) {
+ return this.bytes.subarray(box.start, box.end);
+ }
+}
+
+function find(boxes: Box[], type: string) {
+ const box = boxes.find((b) => b.type === type);
+ if (!box) {
+ throw new Error(`HEIF: missing ${type} box`);
+ }
+ return box;
+}
+
+/** iinf: item id → item type. */
+function itemTypes(reader: Reader, iinf: Box) {
+ const types = new Map<number, string>();
+ const count = reader.uint(reader.open(iinf) === 0 ? 2 : 4);
+ for (const infe of reader.boxes(reader.at, iinf.end).slice(0, count)) {
+ const id = reader.uint(reader.open(infe) === 2 ? 2 : 4);
+ reader.at += 2;
+ types.set(id, reader.fourcc());
+ }
+ return types;
+}
+
+/** iloc: item id → its bytes, joining extents from the file or the idat box. */
+function itemData(reader: Reader, iloc: Box, idat?: Box) {
+ const data = new Map<number, Uint8Array>();
+ const version = reader.open(iloc);
+ const sizes = reader.uint(2);
+ const [offsetSize, lengthSize, baseSize] = [
+ sizes >> 12,
+ (sizes >> 8) & 15,
+ (sizes >> 4) & 15,
+ ];
+ const indexSize = version === 0 ? 0 : sizes & 15;
+ const idSize = version < 2 ? 2 : 4;
+ for (let count = reader.uint(idSize); count > 0; count--) {
+ const id = reader.uint(idSize);
+ const method = version === 0 ? 0 : reader.uint(2) & 15;
+ reader.at += 2;
+ const base =
+ reader.uint(baseSize) + (method === 1 ? (idat?.start ?? 0) : 0);
+ const extents = Array.from({ length: reader.uint(2) }, () => {
+ reader.at += indexSize;
+ const start = base + reader.uint(offsetSize);
+ return reader.bytes.subarray(start, start + reader.uint(lengthSize));
+ });
+ const joined = new Uint8Array(
+ extents.reduce((n, e) => n + e.byteLength, 0),
+ );
+ let at = 0;
+ for (const extent of extents) {
+ joined.set(extent, at);
+ at += extent.byteLength;
+ }
+ data.set(id, joined);
+ }
+ return data;
+}
+
+/** iprp: item id → its property boxes (ipco entries, associated through ipma). */
+function itemProperties(reader: Reader, iprp: Box) {
+ const children = reader.boxes(iprp.start, iprp.end);
+ const ipco = find(children, "ipco");
+ const properties = reader.boxes(ipco.start, ipco.end);
+ const associations = new Map<number, Box[]>();
+ for (const ipma of children.filter((b) => b.type === "ipma")) {
+ reader.at = ipma.start;
+ const versionFlags = reader.uint(4);
+ const wide = versionFlags & 1;
+ for (let count = reader.uint(4); count > 0; count--) {
+ const id = reader.uint(versionFlags >>> 24 === 0 ? 2 : 4);
+ const boxes = Array.from({ length: reader.uint(1) }, () => {
+ const index = reader.uint(wide ? 2 : 1) & (wide ? 0x7fff : 0x7f);
+ return properties[index - 1];
+ });
+ associations.set(
+ id,
+ boxes.filter((b) => b !== undefined),
+ );
+ }
+ }
+ return associations;
+}
+
+/** iref dimg: derived item id → source item ids, in order. */
+function itemSources(reader: Reader, iref?: Box) {
+ const sources = new Map<number, number[]>();
+ if (iref) {
+ const idSize = reader.open(iref) === 0 ? 2 : 4;
+ for (const ref of reader.boxes(reader.at, iref.end)) {
+ reader.at = ref.start;
+ const from = reader.uint(idSize);
+ const to = Array.from({ length: reader.uint(2) }, () =>
+ reader.uint(idSize),
+ );
+ if (ref.type === "dimg") {
+ sources.set(from, to);
+ }
+ }
+ }
+ return sources;
+}
+
+/** WebCodecs codec string for an HEVC decoder configuration record (hvcC). */
+function codecOf(hvcC: Uint8Array) {
+ const profile = hvcC[1] ?? 0;
+ const view = new DataView(hvcC.buffer, hvcC.byteOffset);
+ let compatibility = 0;
+ for (let i = 0; i < 32; i++) {
+ compatibility =
+ ((compatibility << 1) | ((view.getUint32(2) >>> i) & 1)) >>> 0;
+ }
+ const constraints = Array.from(hvcC.subarray(6, 12), (b) =>
+ b.toString(16).padStart(2, "0"),
+ );
+ while (constraints.at(-1) === "00") {
+ constraints.pop();
+ }
+ const space = ["", "A", "B", "C"][profile >> 6];
+ const tier = profile & 0x20 ? "H" : "L";
+ return [
+ `hvc1.${space}${profile & 31}`,
+ compatibility.toString(16),
+ `${tier}${hvcC[12]}`,
+ ...constraints,
+ ].join(".");
+}
+
+/** ispe: [width, height]. */
+function sizeOf(ispe: Uint8Array) {
+ const view = new DataView(ispe.buffer, ispe.byteOffset);
+ return [view.getUint32(4), view.getUint32(8)] as const;
+}
+
+export default async function decodeHeic(file: Blob) {
+ const reader = new Reader(new Uint8Array(await file.arrayBuffer()));
+ const meta = find(reader.boxes(0, reader.bytes.byteLength), "meta");
+ const boxes = reader.boxes(meta.start + 4, meta.end);
+ const primary = reader.uint(reader.open(find(boxes, "pitm")) === 0 ? 2 : 4);
+ const types = itemTypes(reader, find(boxes, "iinf"));
+ const data = itemData(
+ reader,
+ find(boxes, "iloc"),
+ boxes.find((b) => b.type === "idat"),
+ );
+ const properties = itemProperties(reader, find(boxes, "iprp"));
+ const sources = itemSources(
+ reader,
+ boxes.find((b) => b.type === "iref"),
+ );
+ const property = (id: number, type: string) => {
+ const box = properties.get(id)?.find((b) => b.type === type);
+ return box && reader.content(box);
+ };
+
+ const grid = types.get(primary) === "grid" ? data.get(primary) : undefined;
+ const items = grid ? (sources.get(primary) ?? []) : [primary];
+ const hvcC = property(items[0] ?? primary, "hvcC");
+ const ispe = property(primary, "ispe");
+ if (!hvcC || !ispe) {
+ throw new Error("HEIF: primary item is not an HEVC image");
+ }
+ const [width, height] = sizeOf(ispe);
+
+ const tiles: VideoFrame[] = [];
+ await new Promise<void>((resolve, reject) => {
+ const decoder = new VideoDecoder({
+ output: (frame) => {
+ tiles[frame.timestamp] = frame;
+ },
+ error: (error) => {
+ for (const frame of tiles) {
+ frame?.close();
+ }
+ reject(error);
+ },
+ });
+ decoder.configure({ codec: codecOf(hvcC), description: hvcC });
+ items.forEach((id, timestamp) => {
+ decoder.decode(
+ new EncodedVideoChunk({
+ type: "key",
+ timestamp,
+ data: data.get(id) ?? new Uint8Array(),
+ }),
+ );
+ });
+ decoder.flush().then(() => {
+ decoder.close();
+ resolve();
+ }, reject);
+ });
+
+ return {
+ width,
+ height,
+ tiles,
+ columns: grid ? (grid[3] ?? 0) + 1 : 1,
+ rotation: (property(primary, "irot")?.[0] ?? 0) & 3,
+ };
+}
diff --git a/src/lib/decode/index.ts b/src/lib/decode/index.ts
index dff964b..95364ca 100644
--- a/src/lib/decode/index.ts
+++ b/src/lib/decode/index.ts
@@ -1,11 +1,15 @@
import type { Gpu, Target } from "vgpu";
+import decodeHeic from "./heic";
import linearize from "./linearize";
import decodeSvg from "./svg";
import type { Decoder, Pixels } from "./types";
export type { Target };
-/** Decoder backed by a worker module: post the file, receive transferred pixels or an error. */
+/**
+ * Decoder backed by a worker module: post the file, receive transferred pixels or an error.
+ * Unused since HEIC moved to WebCodecs; kept for CPU-heavy decoders to come (RAW).
+ */
export function workerDecoder(
load: () => Promise<{ default: new () => Worker }>,
) {
@@ -33,6 +37,7 @@ type Format = {
const native = async () => createImageBitmap;
const svg = async () => decodeSvg;
+const heic = async () => decodeHeic;
const formats: Format[] = [
{ types: ["image/png"], extensions: ["png"], load: native },
@@ -42,6 +47,11 @@ const formats: Format[] = [
{ types: ["image/avif"], extensions: ["avif"], load: native },
{ types: ["image/bmp"], extensions: ["bmp"], load: native },
{ types: ["image/svg+xml"], extensions: ["svg"], load: svg },
+ {
+ types: ["image/heic", "image/heif"],
+ extensions: ["heic", "heif"],
+ load: heic,
+ },
];
/** Matches by MIME type, then by extension for files the OS doesn't type. */
diff --git a/src/lib/decode/linearize.ts b/src/lib/decode/linearize.ts
index e18fda3..197c420 100644
--- a/src/lib/decode/linearize.ts
+++ b/src/lib/decode/linearize.ts
@@ -2,39 +2,83 @@ import { effect, frame, type Gpu, target } from "vgpu";
import shader from "./linearize.wgsl";
import type { Decoded } from "./types";
-type Size = readonly [number, number];
+type Size = [number, number];
-/** Stages the decoded bytes in a texture whose format decodes them to linear when read. */
-function stage(gpu: Gpu, decoded: Decoded, size: Size) {
- const texture = gpu.device.createTexture({
+/** Display P3 primaries, which TypeScript's VideoColorPrimaries doesn't list yet. */
+const isP3 = (frame?: VideoFrame) =>
+ (frame?.colorSpace.primaries as string) === "smpte432";
+
+/** A texture whose format decodes sRGB-encoded bytes to linear when read. */
+function staging(gpu: Gpu, size: Size) {
+ return gpu.device.createTexture({
size,
format: "rgba8unorm-srgb",
usage: ["texture_binding", "copy_dst", "render_attachment"],
});
+}
+
+/** Uploads the decoded image into a staging texture, tiles laid out on their grid. */
+function stage(gpu: Gpu, decoded: Decoded) {
+ const queue = gpu.gpu.queue;
+ const size: Size = [decoded.width, decoded.height];
if (decoded instanceof ImageBitmap) {
- gpu.gpu.queue.copyExternalImageToTexture(
+ const texture = staging(gpu, size);
+ queue.copyExternalImageToTexture(
{ source: decoded },
{ texture: texture.gpu },
size,
);
decoded.close();
- } else {
- gpu.gpu.queue.writeTexture(
- { texture: texture.gpu },
- decoded.data,
- { bytesPerRow: decoded.width * 4 },
- size,
- );
+ return texture;
+ }
+ if ("tiles" in decoded) {
+ const { tiles, columns } = decoded;
+ const tile: Size = [
+ tiles[0]?.displayWidth ?? 0,
+ tiles[0]?.displayHeight ?? 0,
+ ];
+ const rows = Math.ceil(tiles.length / columns);
+ const texture = staging(gpu, [columns * tile[0], rows * tile[1]]);
+ tiles.forEach((frame, i) => {
+ const origin = [
+ (i % columns) * tile[0],
+ Math.floor(i / columns) * tile[1],
+ ];
+ const colorSpace = isP3(frame) ? "display-p3" : "srgb";
+ queue.copyExternalImageToTexture(
+ { source: frame },
+ { texture: texture.gpu, origin, colorSpace },
+ tile,
+ );
+ frame.close();
+ });
+ return texture;
}
+ const texture = staging(gpu, size);
+ queue.writeTexture(
+ { texture: texture.gpu },
+ decoded.data,
+ { bytesPerRow: decoded.width * 4 },
+ size,
+ );
return texture;
}
/** GPU leg for sRGB-encoded decoders: renders the decoded image into a linear Rec.2020 rgba16float target. */
export default function linearize(gpu: Gpu, decoded: Decoded) {
+ const frames = "tiles" in decoded ? decoded : undefined;
+ const rotation = frames?.rotation ?? 0;
+ const p3 = Number(isP3(frames?.tiles[0]));
const size: Size = [decoded.width, decoded.height];
- const source = stage(gpu, decoded, size);
- const image = target(gpu, { size, format: "rgba16float" });
- frame(gpu, (f) => f.pass(image, effect(gpu, shader, { set: { source } })));
+ const source = stage(gpu, decoded);
+ const image = target(gpu, {
+ size: rotation % 2 ? [size[1], size[0]] : size,
+ format: "rgba16float",
+ });
+ const params = { size, rotation, p3 };
+ frame(gpu, (f) =>
+ f.pass(image, effect(gpu, shader, { set: { source, params } })),
+ );
source.dispose();
return image;
}
diff --git a/src/lib/decode/linearize.wgsl b/src/lib/decode/linearize.wgsl
index 6adac06..7bb7ebc 100644
--- a/src/lib/decode/linearize.wgsl
+++ b/src/lib/decode/linearize.wgsl
@@ -1,8 +1,23 @@
-import { srgbToRec2020 } from "../color.wgsl";
+import { p3ToRec2020, srgbToRec2020 } from "../color.wgsl";
-@group(0) @binding(0) var source: texture_2d<f32>;
+struct Params {
+ size: vec2f,
+ rotation: u32,
+ p3: u32,
+}
+@group(0) @binding(0) var<uniform> params: Params;
+@group(0) @binding(1) var source: texture_2d<f32>;
-@fragment fn fs_main(@builtin(position) position: vec4f) -> @location(0) vec4f {
- let texel = textureLoad(source, vec2i(position.xy), 0);
- return vec4f(srgbToRec2020 * texel.rgb, texel.a);
+@fragment fn fs_main(@location(0) uv: vec2f) -> @location(0) vec4f {
+ // Undo `rotation` counter-clockwise quarter turns: where this output pixel sits in the source.
+ var p = uv;
+ switch (params.rotation) {
+ case 1u: { p = vec2f(1.0 - uv.y, uv.x); }
+ case 2u: { p = 1.0 - uv; }
+ case 3u: { p = vec2f(uv.y, 1.0 - uv.x); }
+ default: {}
+ }
+ let texel = textureLoad(source, vec2i(p * params.size), 0);
+ let color = select(srgbToRec2020 * texel.rgb, p3ToRec2020 * texel.rgb, params.p3 == 1u);
+ return vec4f(color, texel.a);
}
diff --git a/src/lib/decode/types.ts b/src/lib/decode/types.ts
index 823c0da..c693006 100644
--- a/src/lib/decode/types.ts
+++ b/src/lib/decode/types.ts
@@ -4,5 +4,13 @@ export type Pixels = {
height: number;
data: Uint8ClampedArray<ArrayBuffer>;
};
-export type Decoded = ImageBitmap | Pixels;
+/** Video frames tiled on a grid `columns` wide, cropped to width × height, then rotated counter-clockwise by `rotation` quarter turns. */
+export type Frames = {
+ width: number;
+ height: number;
+ tiles: VideoFrame[];
+ columns: number;
+ rotation: number;
+};
+export type Decoded = ImageBitmap | Pixels | Frames;
export type Decoder = (file: Blob) => Promise<Decoded>;