Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add audio widget (#3863)

Chenlei Hu3b423afccab8
diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.py
index 57d0a20ab..8efa7c0a8 100644
--- a/comfy_extras/nodes_audio.py
+++ b/comfy_extras/nodes_audio.py
@@ -93,11 +93,18 @@ class SaveAudio:
return { "ui": { "audio": results } }
class LoadAudio:
+ SUPPORTED_FORMATS = ('.wav', '.mp3', '.ogg', '.flac', '.aiff', '.aif')
+
@classmethod
def INPUT_TYPES(s):
input_dir = folder_paths.get_input_directory()
- files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
- return {"required": {"audio": [sorted(files), ]}, }
+ files = [
+ f for f in os.listdir(input_dir)
+ if (os.path.isfile(os.path.join(input_dir, f))
+ and f.endswith(LoadAudio.SUPPORTED_FORMATS)
+ )
+ ]
+ return {"required": {"audio": (sorted(files), {"audio_upload": True})}}
CATEGORY = "_for_testing/audio"
diff --git a/web/extensions/core/uploadAudio.js b/web/extensions/core/uploadAudio.js
new file mode 100644
index 000000000..d3dfef4a7
--- /dev/null
+++ b/web/extensions/core/uploadAudio.js
@@ -0,0 +1,176 @@
+import { app } from "../../scripts/app.js"
+import { api } from "../../scripts/api.js"
+
+function splitFilePath(path) {
+ const folder_separator = path.lastIndexOf("/")
+ if (folder_separator === -1) {
+ return ["", path]
+ }
+ return [
+ path.substring(0, folder_separator),
+ path.substring(folder_separator + 1)
+ ]
+}
+
+function getResourceURL(subfolder, filename, type = "input") {
+ const params = [
+ "filename=" + encodeURIComponent(filename),
+ "type=" + type,
+ "subfolder=" + subfolder,
+ app.getPreviewFormatParam().substring(1),
+ app.getRandParam().substring(1)
+ ].join("&")
+
+ return `/view?${params}`
+}
+
+async function uploadFile(
+ audioWidget,
+ audioUIWidget,
+ file,
+ updateNode,
+ pasted = false
+) {
+ try {
+ // Wrap file in formdata so it includes filename
+ const body = new FormData()
+ body.append("image", file)
+ if (pasted) body.append("subfolder", "pasted")
+ const resp = await api.fetchApi("/upload/image", {
+ method: "POST",
+ body
+ })
+
+ if (resp.status === 200) {
+ const data = await resp.json()
+ // Add the file to the dropdown list and update the widget value
+ let path = data.name
+ if (data.subfolder) path = data.subfolder + "/" + path
+
+ if (!audioWidget.options.values.includes(path)) {
+ audioWidget.options.values.push(path)
+ }
+
+ if (updateNode) {
+ audioUIWidget.element.src = api.apiURL(
+ getResourceURL(...splitFilePath(path))
+ )
+ audioWidget.value = path
+ }
+ } else {
+ alert(resp.status + " - " + resp.statusText)
+ }
+ } catch (error) {
+ alert(error)
+ }
+}
+
+// AudioWidget MUST be registered first, as AUDIOUPLOAD depends on AUDIO_UI to be
+// present.
+app.registerExtension({
+ name: "Comfy.AudioWidget",
+ async beforeRegisterNodeDef(nodeType, nodeData) {
+ if (["LoadAudio", "SaveAudio"].includes(nodeType.comfyClass)) {
+ nodeData.input.required.audioUI = ["AUDIO_UI"]
+ }
+ },
+ getCustomWidgets() {
+ return {
+ AUDIO_UI(node, inputName) {
+ const audio = document.createElement("audio")
+ audio.controls = true
+ audio.classList.add("comfy-audio")
+ audio.setAttribute("name", "media")
+
+ const audioUIWidget = node.addDOMWidget(
+ inputName,
+ /* name=*/ "audioUI",
+ audio
+ )
+ // @ts-ignore
+ // TODO: Sort out the DOMWidget type.
+ audioUIWidget.serialize = false
+
+ const isOutputNode = node.constructor.nodeData.output_node
+ if (isOutputNode) {
+ // Hide the audio widget when there is no audio initially.
+ audioUIWidget.element.classList.add("empty-audio-widget")
+ // Populate the audio widget UI on node execution.
+ const onExecuted = node.onExecuted
+ node.onExecuted = function(message) {
+ onExecuted?.apply(this, arguments)
+ const audios = message.audio
+ if (!audios) return
+ const audio = audios[0]
+ audioUIWidget.element.src = api.apiURL(
+ getResourceURL(audio.subfolder, audio.filename, "output")
+ )
+ audioUIWidget.element.classList.remove("empty-audio-widget")
+ }
+ }
+ return { widget: audioUIWidget }
+ }
+ }
+ },
+ onNodeOutputsUpdated(nodeOutputs) {
+ for (const [nodeId, output] of Object.entries(nodeOutputs)) {
+ const node = app.graph.getNodeById(Number.parseInt(nodeId));
+ if ("audio" in output) {
+ const audioUIWidget = node.widgets.find((w) => w.name === "audioUI");
+ const audio = output.audio[0];
+ audioUIWidget.element.src = api.apiURL(getResourceURL(audio.subfolder, audio.filename, "output"));
+ audioUIWidget.element.classList.remove("empty-audio-widget");
+ }
+ }
+ },
+})
+
+app.registerExtension({
+ name: "Comfy.UploadAudio",
+ async beforeRegisterNodeDef(nodeType, nodeData) {
+ if (nodeData?.input?.required?.audio?.[1]?.audio_upload === true) {
+ nodeData.input.required.upload = ["AUDIOUPLOAD"]
+ }
+ },
+ getCustomWidgets() {
+ return {
+ AUDIOUPLOAD(node, inputName) {
+ // The widget that allows user to select file.
+ const audioWidget = node.widgets.find(w => w.name === "audio")
+ const audioUIWidget = node.widgets.find(w => w.name === "audioUI")
+
+ const onAudioWidgetUpdate = () => {
+ audioUIWidget.element.src = api.apiURL(
+ getResourceURL(...splitFilePath(audioWidget.value))
+ )
+ }
+ // Initially load default audio file to audioUIWidget.
+ onAudioWidgetUpdate()
+ audioWidget.callback = onAudioWidgetUpdate
+
+ const fileInput = document.createElement("input")
+ fileInput.type = "file"
+ fileInput.accept = "audio/*"
+ fileInput.style.display = "none"
+ fileInput.onchange = () => {
+ if (fileInput.files.length) {
+ uploadFile(audioWidget, audioUIWidget, fileInput.files[0], true)
+ }
+ }
+ // The widget to pop up the upload dialog.
+ const uploadWidget = node.addWidget(
+ "button",
+ inputName,
+ /* value=*/ "",
+ () => {
+ fileInput.click()
+ }
+ )
+ uploadWidget.label = "choose file to upload"
+ uploadWidget.serialize = false
+
+ return { widget: uploadWidget }
+ }
+ }
+ }
+})
diff --git a/web/scripts/app.js b/web/scripts/app.js
index a354ff805..5fc9b9c6c 100644
--- a/web/scripts/app.js
+++ b/web/scripts/app.js
@@ -71,7 +71,7 @@ export class ComfyApp {
* Stores the execution output data for each node
* @type {Record<string, any>}
*/
- this.nodeOutputs = {};
+ this._nodeOutputs = {};
/**
* Stores the preview image data for each node
@@ -86,6 +86,15 @@ export class ComfyApp {
this.shiftDown = false;
}
+ get nodeOutputs() {
+ return this._nodeOutputs;
+ }
+
+ set nodeOutputs(value) {
+ this._nodeOutputs = value;
+ this.#invokeExtensions("onNodeOutputsUpdated", value);
+ }
+
getPreviewFormatParam() {
let preview_format = this.ui.settings.getSettingValue("Comfy.PreviewFormat");
if(preview_format)
diff --git a/web/style.css b/web/style.css
index 8091a489f..e983b652a 100644
--- a/web/style.css
+++ b/web/style.css
@@ -632,3 +632,7 @@ dialog::backdrop {
border-top: none;
}
}
+
+audio.comfy-audio.empty-audio-widget {
+ display: none;
+}