Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

improve: Mask Editor (#2171)

Dr.Lt.Data8112a0d9fcb8
diff --git a/web/extensions/core/maskeditor.js b/web/extensions/core/maskeditor.js
index 8ace79562..1ea4dbcaa 100644
--- a/web/extensions/core/maskeditor.js
+++ b/web/extensions/core/maskeditor.js
@@ -33,6 +33,18 @@ function loadedImageToBlob(image) {
return blob;
}
+function loadImage(imagePath) {
+ return new Promise((resolve, reject) => {
+ const image = new Image();
+
+ image.onload = function() {
+ resolve(image);
+ };
+
+ image.src = imagePath;
+ });
+}
+
async function uploadMask(filepath, formData) {
await api.fetchApi('/upload/mask', {
method: 'POST',
@@ -50,25 +62,25 @@ async function uploadMask(filepath, formData) {
ClipspaceDialog.invalidatePreview();
}
-function prepareRGB(image, backupCanvas, backupCtx) {
+function prepare_mask(image, maskCanvas, maskCtx) {
// paste mask data into alpha channel
- backupCtx.drawImage(image, 0, 0, backupCanvas.width, backupCanvas.height);
- const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
+ maskCtx.drawImage(image, 0, 0, maskCanvas.width, maskCanvas.height);
+ const maskData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
- // refine mask image
- for (let i = 0; i < backupData.data.length; i += 4) {
- if(backupData.data[i+3] == 255)
- backupData.data[i+3] = 0;
+ // invert mask
+ for (let i = 0; i < maskData.data.length; i += 4) {
+ if(maskData.data[i+3] == 255)
+ maskData.data[i+3] = 0;
else
- backupData.data[i+3] = 255;
+ maskData.data[i+3] = 255;
- backupData.data[i] = 0;
- backupData.data[i+1] = 0;
- backupData.data[i+2] = 0;
+ maskData.data[i] = 0;
+ maskData.data[i+1] = 0;
+ maskData.data[i+2] = 0;
}
- backupCtx.globalCompositeOperation = 'source-over';
- backupCtx.putImageData(backupData, 0, 0);
+ maskCtx.globalCompositeOperation = 'source-over';
+ maskCtx.putImageData(maskData, 0, 0);
}
class MaskEditorDialog extends ComfyDialog {
@@ -184,14 +196,13 @@ class MaskEditorDialog extends ComfyDialog {
this.element.appendChild(bottom_panel);
document.body.appendChild(brush);
- var brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
+ this.brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
self.brush_size = event.target.value;
self.updateBrushPreview(self, null, null);
});
var clearButton = this.createLeftButton("Clear",
() => {
self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
- self.backupCtx.clearRect(0, 0, self.backupCanvas.width, self.backupCanvas.height);
});
var cancelButton = this.createRightButton("Cancel", () => {
document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
@@ -213,34 +224,37 @@ class MaskEditorDialog extends ComfyDialog {
bottom_panel.appendChild(clearButton);
bottom_panel.appendChild(this.saveButton);
bottom_panel.appendChild(cancelButton);
- bottom_panel.appendChild(brush_size_slider);
+ bottom_panel.appendChild(this.brush_size_slider);
+
+ imgCanvas.style.position = "absolute";
+ maskCanvas.style.position = "absolute";
- imgCanvas.style.position = "relative";
imgCanvas.style.top = "200";
imgCanvas.style.left = "0";
- maskCanvas.style.position = "absolute";
+ maskCanvas.style.top = imgCanvas.style.top;
+ maskCanvas.style.left = imgCanvas.style.left;
}
- show() {
+ async show() {
+ this.zoom_ratio = 1.0;
+ this.pan_x = 0;
+ this.pan_y = 0;
+
if(!this.is_layout_created) {
// layout
const imgCanvas = document.createElement('canvas');
const maskCanvas = document.createElement('canvas');
- const backupCanvas = document.createElement('canvas');
imgCanvas.id = "imageCanvas";
maskCanvas.id = "maskCanvas";
- backupCanvas.id = "backupCanvas";
this.setlayout(imgCanvas, maskCanvas);
// prepare content
this.imgCanvas = imgCanvas;
this.maskCanvas = maskCanvas;
- this.backupCanvas = backupCanvas;
- this.maskCtx = maskCanvas.getContext('2d');
- this.backupCtx = backupCanvas.getContext('2d');
+ this.maskCtx = maskCanvas.getContext('2d', {willReadFrequently: true });
this.setEventHandler(maskCanvas);
@@ -252,6 +266,8 @@ class MaskEditorDialog extends ComfyDialog {
mutations.forEach(function(mutation) {
if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
+ document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
+ self.brush.style.display = "none";
ComfyApp.onClipspaceEditorClosed();
}
@@ -264,7 +280,8 @@ class MaskEditorDialog extends ComfyDialog {
observer.observe(this.element, config);
}
- this.setImages(this.imgCanvas, this.backupCanvas);
+ // The keydown event needs to be reconfigured when closing the dialog as it gets removed.
+ document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
if(ComfyApp.clipspace_return_node) {
this.saveButton.innerText = "Save to node";
@@ -275,97 +292,157 @@ class MaskEditorDialog extends ComfyDialog {
this.saveButton.disabled = false;
this.element.style.display = "block";
+ this.element.style.width = "85%";
+ this.element.style.margin = "0 7.5%";
+ this.element.style.height = "100vh";
+ this.element.style.top = "50%";
+ this.element.style.left = "42%";
this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
+
+ await this.setImages(this.imgCanvas);
+
+ this.is_visible = true;
}
isOpened() {
return this.element.style.display == "block";
}
- setImages(imgCanvas, backupCanvas) {
- const imgCtx = imgCanvas.getContext('2d');
- const backupCtx = backupCanvas.getContext('2d');
+ invalidateCanvas(orig_image, mask_image) {
+ this.imgCanvas.width = orig_image.width;
+ this.imgCanvas.height = orig_image.height;
+
+ this.maskCanvas.width = orig_image.width;
+ this.maskCanvas.height = orig_image.height;
+
+ let imgCtx = this.imgCanvas.getContext('2d', {willReadFrequently: true });
+ let maskCtx = this.maskCanvas.getContext('2d', {willReadFrequently: true });
+
+ imgCtx.drawImage(orig_image, 0, 0, orig_image.width, orig_image.height);
+ prepare_mask(mask_image, this.maskCanvas, maskCtx);
+ }
+
+ async setImages(imgCanvas) {
+ let self = this;
+
+ const imgCtx = imgCanvas.getContext('2d', {willReadFrequently: true });
const maskCtx = this.maskCtx;
const maskCanvas = this.maskCanvas;
- backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
imgCtx.clearRect(0,0,this.imgCanvas.width,this.imgCanvas.height);
maskCtx.clearRect(0,0,this.maskCanvas.width,this.maskCanvas.height);
// image load
- const orig_image = new Image();
- window.addEventListener("resize", () => {
- // repositioning
- imgCanvas.width = window.innerWidth - 250;
- imgCanvas.height = window.innerHeight - 200;
-
- // redraw image
- let drawWidth = orig_image.width;
- let drawHeight = orig_image.height;
- if (orig_image.width > imgCanvas.width) {
- drawWidth = imgCanvas.width;
- drawHeight = (drawWidth / orig_image.width) * orig_image.height;
- }
-
- if (drawHeight > imgCanvas.height) {
- drawHeight = imgCanvas.height;
- drawWidth = (drawHeight / orig_image.height) * orig_image.width;
- }
-
- imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight);
-
- // update mask
- maskCanvas.width = drawWidth;
- maskCanvas.height = drawHeight;
- maskCanvas.style.top = imgCanvas.offsetTop + "px";
- maskCanvas.style.left = imgCanvas.offsetLeft + "px";
- backupCtx.drawImage(maskCanvas, 0, 0, maskCanvas.width, maskCanvas.height, 0, 0, backupCanvas.width, backupCanvas.height);
- maskCtx.drawImage(backupCanvas, 0, 0, backupCanvas.width, backupCanvas.height, 0, 0, maskCanvas.width, maskCanvas.height);
- });
-
const filepath = ComfyApp.clipspace.images;
- const touched_image = new Image();
-
- touched_image.onload = function() {
- backupCanvas.width = touched_image.width;
- backupCanvas.height = touched_image.height;
-
- prepareRGB(touched_image, backupCanvas, backupCtx);
- };
-
const alpha_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src)
alpha_url.searchParams.delete('channel');
alpha_url.searchParams.delete('preview');
alpha_url.searchParams.set('channel', 'a');
- touched_image.src = alpha_url;
+ let mask_image = await loadImage(alpha_url);
// original image load
- orig_image.onload = function() {
- window.dispatchEvent(new Event('resize'));
- };
-
const rgb_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src);
rgb_url.searchParams.delete('channel');
rgb_url.searchParams.set('channel', 'rgb');
- orig_image.src = rgb_url;
- this.image = orig_image;
+ this.image = new Image();
+ this.image.onload = function() {
+ maskCanvas.width = self.image.width;
+ maskCanvas.height = self.image.height;
+
+ self.invalidateCanvas(self.image, mask_image);
+ self.initializeCanvasPanZoom();
+ };
+ this.image.src = rgb_url;
}
- setEventHandler(maskCanvas) {
- maskCanvas.addEventListener("contextmenu", (event) => {
- event.preventDefault();
- });
+ initializeCanvasPanZoom() {
+ // set initialize
+ let drawWidth = this.image.width;
+ let drawHeight = this.image.height;
+
+ let width = this.element.clientWidth;
+ let height = this.element.clientHeight;
+
+ if (this.image.width > width) {
+ drawWidth = width;
+ drawHeight = (drawWidth / this.image.width) * this.image.height;
+ }
+
+ if (drawHeight > height) {
+ drawHeight = height;
+ drawWidth = (drawHeight / this.image.height) * this.image.width;
+ }
+
+ this.zoom_ratio = drawWidth/this.image.width;
+
+ const canvasX = (width - drawWidth) / 2;
+ const canvasY = (height - drawHeight) / 2;
+ this.pan_x = canvasX;
+ this.pan_y = canvasY;
+
+ this.invalidatePanZoom();
+ }
+
+
+ invalidatePanZoom() {
+ let raw_width = this.image.width * this.zoom_ratio;
+ let raw_height = this.image.height * this.zoom_ratio;
+
+ if(this.pan_x + raw_width < 10) {
+ this.pan_x = 10 - raw_width;
+ }
+
+ if(this.pan_y + raw_height < 10) {
+ this.pan_y = 10 - raw_height;
+ }
+
+ let width = `${raw_width}px`;
+ let height = `${raw_height}px`;
+ let left = `${this.pan_x}px`;
+ let top = `${this.pan_y}px`;
+
+ this.maskCanvas.style.width = width;
+ this.maskCanvas.style.height = height;
+ this.maskCanvas.style.left = left;
+ this.maskCanvas.style.top = top;
+
+ this.imgCanvas.style.width = width;
+ this.imgCanvas.style.height = height;
+ this.imgCanvas.style.left = left;
+ this.imgCanvas.style.top = top;
+ }
+
+
+ setEventHandler(maskCanvas) {
const self = this;
- maskCanvas.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
- maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
- document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
- maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
- maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
- maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
- maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
- document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
+
+ if(!this.handler_registered) {
+ maskCanvas.addEventListener("contextmenu", (event) => {
+ event.preventDefault();
+ });
+
+ this.element.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
+ this.element.addEventListener('pointermove', (event) => this.pointMoveEvent(self,event));
+ this.element.addEventListener('touchmove', (event) => this.pointMoveEvent(self,event));
+
+ this.element.addEventListener('dragstart', (event) => {
+ if(event.ctrlKey) {
+ event.preventDefault();
+ }
+ });
+
+ maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
+ maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
+ maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
+ maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
+ maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
+
+ document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
+
+ this.handler_registered = true;
+ }
}
brush_size = 10;
@@ -378,8 +455,10 @@ class MaskEditorDialog extends ComfyDialog {
const self = MaskEditorDialog.instance;
if (event.key === ']') {
self.brush_size = Math.min(self.brush_size+2, 100);
+ self.brush_slider_input.value = self.brush_size;
} else if (event.key === '[') {
self.brush_size = Math.max(self.brush_size-2, 1);
+ self.brush_slider_input.value = self.brush_size;
} else if(event.key === 'Enter') {
self.save();
}
@@ -389,6 +468,10 @@ class MaskEditorDialog extends ComfyDialog {
static handlePointerUp(event) {
event.preventDefault();
+
+ this.mousedown_x = null;
+ this.mousedown_y = null;
+
MaskEditorDialog.instance.drawing_mode = false;
}
@@ -398,24 +481,70 @@ class MaskEditorDialog extends ComfyDialog {
var centerX = self.cursorX;
var centerY = self.cursorY;
- brush.style.width = self.brush_size * 2 + "px";
- brush.style.height = self.brush_size * 2 + "px";
- brush.style.left = (centerX - self.brush_size) + "px";
- brush.style.top = (centerY - self.brush_size) + "px";
+ brush.style.width = self.brush_size * 2 * this.zoom_ratio + "px";
+ brush.style.height = self.brush_size * 2 * this.zoom_ratio + "px";
+ brush.style.left = (centerX - self.brush_size * this.zoom_ratio) + "px";
+ brush.style.top = (centerY - self.brush_size * this.zoom_ratio) + "px";
}
handleWheelEvent(self, event) {
- if(event.deltaY < 0)
- self.brush_size = Math.min(self.brush_size+2, 100);
- else
- self.brush_size = Math.max(self.brush_size-2, 1);
+ event.preventDefault();
- self.brush_slider_input.value = self.brush_size;
+ if(event.ctrlKey) {
+ // zoom canvas
+ if(event.deltaY < 0) {
+ this.zoom_ratio = Math.min(10.0, this.zoom_ratio+0.2);
+ }
+ else {
+ this.zoom_ratio = Math.max(0.2, this.zoom_ratio-0.2);
+ }
+
+ this.invalidatePanZoom();
+ }
+ else {
+ // adjust brush size
+ if(event.deltaY < 0)
+ this.brush_size = Math.min(this.brush_size+2, 100);
+ else
+ this.brush_size = Math.max(this.brush_size-2, 1);
+
+ this.brush_slider_input.value = this.brush_size;
+
+ this.updateBrushPreview(this);
+ }
+ }
+
+ pointMoveEvent(self, event) {
+ this.cursorX = event.pageX;
+ this.cursorY = event.pageY;
self.updateBrushPreview(self);
+
+ if(event.ctrlKey) {
+ event.preventDefault();
+ self.pan_move(self, event);
+ }
+ }
+
+ pan_move(self, event) {
+ if(event.buttons == 1) {
+ if(this.mousedown_x) {
+ let deltaX = this.mousedown_x - event.clientX;
+ let deltaY = this.mousedown_y - event.clientY;
+
+ self.pan_x = this.mousedown_pan_x - deltaX;
+ self.pan_y = this.mousedown_pan_y - deltaY;
+
+ self.invalidatePanZoom();
+ }
+ }
}
draw_move(self, event) {
+ if(event.ctrlKey) {
+ return;
+ }
+
event.preventDefault();
this.cursorX = event.pageX;
@@ -439,6 +568,9 @@ class MaskEditorDialog extends ComfyDialog {
y = event.targetTouches[0].clientY - maskRect.top;
}
+ x /= self.zoom_ratio;
+ y /= self.zoom_ratio;
+
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
brush_size *= event.pressure;
@@ -489,8 +621,8 @@ class MaskEditorDialog extends ComfyDialog {
}
else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
const maskRect = self.maskCanvas.getBoundingClientRect();
- const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
- const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+ const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
+ const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
@@ -540,6 +672,17 @@ class MaskEditorDialog extends ComfyDialog {
}
handlePointerDown(self, event) {
+ if(event.ctrlKey) {
+ if (event.buttons == 1) {
+ this.mousedown_x = event.clientX;
+ this.mousedown_y = event.clientY;
+
+ this.mousedown_pan_x = this.pan_x;
+ this.mousedown_pan_y = this.pan_y;
+ }
+ return;
+ }
+
var brush_size = this.brush_size;
if(event instanceof PointerEvent && event.pointerType == 'pen') {
brush_size *= event.pressure;
@@ -551,8 +694,8 @@ class MaskEditorDialog extends ComfyDialog {
event.preventDefault();
const maskRect = self.maskCanvas.getBoundingClientRect();
- const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
- const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+ const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
+ const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
self.maskCtx.beginPath();
if (event.button == 0) {
@@ -570,15 +713,18 @@ class MaskEditorDialog extends ComfyDialog {
}
async save() {
- const backupCtx = this.backupCanvas.getContext('2d', {willReadFrequently:true});
+ const backupCanvas = document.createElement('canvas');
+ const backupCtx = backupCanvas.getContext('2d', {willReadFrequently:true});
+ backupCanvas.width = this.image.width;
+ backupCanvas.height = this.image.height;
- backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
+ backupCtx.clearRect(0,0, backupCanvas.width, backupCanvas.height);
backupCtx.drawImage(this.maskCanvas,
0, 0, this.maskCanvas.width, this.maskCanvas.height,
- 0, 0, this.backupCanvas.width, this.backupCanvas.height);
+ 0, 0, backupCanvas.width, backupCanvas.height);
// paste mask data into alpha channel
- const backupData = backupCtx.getImageData(0, 0, this.backupCanvas.width, this.backupCanvas.height);
+ const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
// refine mask image
for (let i = 0; i < backupData.data.length; i += 4) {
@@ -615,7 +761,7 @@ class MaskEditorDialog extends ComfyDialog {
ComfyApp.clipspace.widgets[index].value = item;
}
- const dataURL = this.backupCanvas.toDataURL();
+ const dataURL = backupCanvas.toDataURL();
const blob = dataURLToBlob(dataURL);
let original_url = new URL(this.image.src);