The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Ci quality workflows (#1423)
enzymezoo-code26cd8405ddb3
diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.ymlnew file mode 100644index 000000000..421dd5ee4--- /dev/null+++ b/.github/workflows/test-build.yml@@ -0,0 +1,31 @@+name: Build package++#+# This workflow is a test of the python package build.+# Install Python dependencies across different Python versions.+#++on:+ push:+ paths:+ - "requirements.txt"+ - ".github/workflows/test-build.yml"++jobs:+ build:+ name: Build Test+ runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ python-version: ["3.8", "3.9", "3.10", "3.11"]+ steps:+ - uses: actions/checkout@v2+ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v2+ with:+ python-version: ${{ matrix.python-version }}+ - name: Install dependencies+ run: |+ python -m pip install --upgrade pip+ pip install -r requirements.txt\ No newline at end of filediff --git a/pytest.ini b/pytest.ininew file mode 100644index 000000000..b5a68e0f1--- /dev/null+++ b/pytest.ini@@ -0,0 +1,5 @@+[pytest]+markers =+ inference: mark as inference test (deselect with '-m "not inference"')+testpaths = tests+addopts = -s\ No newline at end of filediff --git a/tests/README.md b/tests/README.mdnew file mode 100644index 000000000..2005fd45b--- /dev/null+++ b/tests/README.md@@ -0,0 +1,29 @@+# Automated Testing++## Running tests locally++Additional requirements for running tests:+```+pip install pytest+pip install websocket-client==1.6.1+opencv-python==4.6.0.66+scikit-image==0.21.0+```+Run inference tests:+```+pytest tests/inference+```++## Quality regression test+Compares images in 2 directories to ensure they are the same++1) Run an inference test to save a directory of "ground truth" images+```+ pytest tests/inference --output_dir tests/inference/baseline+```+2) Make code edits++3) Run inference and quality comparison tests+```+pytest+```\ No newline at end of filediff --git a/tests/__init__.py b/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/compare/conftest.py b/tests/compare/conftest.pynew file mode 100644index 000000000..dd5078c9e--- /dev/null+++ b/tests/compare/conftest.py@@ -0,0 +1,41 @@+import os+import pytest++# Command line arguments for pytest+def pytest_addoption(parser):+ parser.addoption('--baseline_dir', action="store", default='tests/inference/baseline', help='Directory for ground-truth images')+ parser.addoption('--test_dir', action="store", default='tests/inference/samples', help='Directory for images to test')+ parser.addoption('--metrics_file', action="store", default='tests/metrics.md', help='Output file for metrics')+ parser.addoption('--img_output_dir', action="store", default='tests/compare/samples', help='Output directory for diff metric images')++# This initializes args at the beginning of the test session[email protected](scope="session", autouse=True)+def args_pytest(pytestconfig):+ args = {}+ args['baseline_dir'] = pytestconfig.getoption('baseline_dir')+ args['test_dir'] = pytestconfig.getoption('test_dir')+ args['metrics_file'] = pytestconfig.getoption('metrics_file')+ args['img_output_dir'] = pytestconfig.getoption('img_output_dir')++ # Initialize metrics file+ with open(args['metrics_file'], 'a') as f:+ # if file is empty, write header+ if os.stat(args['metrics_file']).st_size == 0:+ f.write("| date | run | file | status | value | \n")+ f.write("| --- | --- | --- | --- | --- | \n")++ return args+++def gather_file_basenames(directory: str):+ files = []+ for file in os.listdir(directory):+ if file.endswith(".png"):+ files.append(file)+ return files++# Creates the list of baseline file names to use as a fixture+def pytest_generate_tests(metafunc):+ if "baseline_fname" in metafunc.fixturenames:+ baseline_fnames = gather_file_basenames(metafunc.config.getoption("baseline_dir"))+ metafunc.parametrize("baseline_fname", baseline_fnames)diff --git a/tests/compare/test_quality.py b/tests/compare/test_quality.pynew file mode 100644index 000000000..92a2d5a8b--- /dev/null+++ b/tests/compare/test_quality.py@@ -0,0 +1,195 @@+import datetime+import numpy as np+import os+from PIL import Image+import pytest+from pytest import fixture+from typing import Tuple, List++from cv2 import imread, cvtColor, COLOR_BGR2RGB+from skimage.metrics import structural_similarity as ssim+++"""+This test suite compares images in 2 directories by file name+The directories are specified by the command line arguments --baseline_dir and --test_dir++"""+# ssim: Structural Similarity Index+# Returns a tuple of (ssim, diff_image)+def ssim_score(img0: np.ndarray, img1: np.ndarray) -> Tuple[float, np.ndarray]:+ score, diff = ssim(img0, img1, channel_axis=-1, full=True)+ # rescale the difference image to 0-255 range+ diff = (diff * 255).astype("uint8")+ return score, diff++# Metrics must return a tuple of (score, diff_image)+METRICS = {"ssim": ssim_score}+METRICS_PASS_THRESHOLD = {"ssim": 0.95}+++class TestCompareImageMetrics:+ @fixture(scope="class")+ def test_file_names(self, args_pytest):+ test_dir = args_pytest['test_dir']+ fnames = self.gather_file_basenames(test_dir)+ yield fnames+ del fnames++ @fixture(scope="class", autouse=True)+ def teardown(self, args_pytest):+ yield+ # Runs after all tests are complete+ # Aggregate output files into a grid of images+ baseline_dir = args_pytest['baseline_dir']+ test_dir = args_pytest['test_dir']+ img_output_dir = args_pytest['img_output_dir']+ metrics_file = args_pytest['metrics_file']++ grid_dir = os.path.join(img_output_dir, "grid")+ os.makedirs(grid_dir, exist_ok=True)++ for metric_dir in METRICS.keys():+ metric_path = os.path.join(img_output_dir, metric_dir)+ for file in os.listdir(metric_path):+ if file.endswith(".png"):+ score = self.lookup_score_from_fname(file, metrics_file)+ image_file_list = []+ image_file_list.append([+ os.path.join(baseline_dir, file),+ os.path.join(test_dir, file),+ os.path.join(metric_path, file)+ ])+ # Create grid+ image_list = [[Image.open(file) for file in files] for files in image_file_list]+ grid = self.image_grid(image_list)+ grid.save(os.path.join(grid_dir, f"{metric_dir}_{score:.3f}_{file}"))++ # Tests run for each baseline file name+ @fixture()+ def fname(self, baseline_fname):+ yield baseline_fname+ del baseline_fname++ def test_directories_not_empty(self, args_pytest):+ baseline_dir = args_pytest['baseline_dir']+ test_dir = args_pytest['test_dir']+ assert len(os.listdir(baseline_dir)) != 0, f"Baseline directory {baseline_dir} is empty"+ assert len(os.listdir(test_dir)) != 0, f"Test directory {test_dir} is empty"++ def test_dir_has_all_matching_metadata(self, fname, test_file_names, args_pytest):+ # Check that all files in baseline_dir have a file in test_dir with matching metadata+ baseline_file_path = os.path.join(args_pytest['baseline_dir'], fname)+ file_paths = [os.path.join(args_pytest['test_dir'], f) for f in test_file_names]+ file_match = self.find_file_match(baseline_file_path, file_paths)+ assert file_match is not None, f"Could not find a file in {args_pytest['test_dir']} with matching metadata to {baseline_file_path}"++ # For a baseline image file, finds the corresponding file name in test_dir and+ # compares the images using the metrics in METRICS+ @pytest.mark.parametrize("metric", METRICS.keys())+ def test_pipeline_compare(+ self,+ args_pytest,+ fname,+ test_file_names,+ metric,+ ):+ baseline_dir = args_pytest['baseline_dir']+ test_dir = args_pytest['test_dir']+ metrics_output_file = args_pytest['metrics_file']+ img_output_dir = args_pytest['img_output_dir']++ baseline_file_path = os.path.join(baseline_dir, fname)++ # Find file match+ file_paths = [os.path.join(test_dir, f) for f in test_file_names]+ test_file = self.find_file_match(baseline_file_path, file_paths)++ # Run metrics+ sample_baseline = self.read_img(baseline_file_path)+ sample_secondary = self.read_img(test_file)++ score, metric_img = METRICS[metric](sample_baseline, sample_secondary)+ metric_status = score > METRICS_PASS_THRESHOLD[metric]++ # Save metric values+ with open(metrics_output_file, 'a') as f:+ run_info = os.path.splitext(fname)[0]+ metric_status_str = "PASS ✅" if metric_status else "FAIL ❌"+ date_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+ f.write(f"| {date_str} | {run_info} | {metric} | {metric_status_str} | {score} | \n")++ # Save metric image+ metric_img_dir = os.path.join(img_output_dir, metric)+ os.makedirs(metric_img_dir, exist_ok=True)+ output_filename = f'{fname}'+ Image.fromarray(metric_img).save(os.path.join(metric_img_dir, output_filename))++ assert score > METRICS_PASS_THRESHOLD[metric]++ def read_img(self, filename: str) -> np.ndarray:+ cvImg = imread(filename)+ cvImg = cvtColor(cvImg, COLOR_BGR2RGB)+ return cvImg++ def image_grid(self, img_list: list[list[Image.Image]]):+ # imgs is a 2D list of images+ # Assumes the input images are a rectangular grid of equal sized images+ rows = len(img_list)+ cols = len(img_list[0])++ w, h = img_list[0][0].size+ grid = Image.new('RGB', size=(cols*w, rows*h))++ for i, row in enumerate(img_list):+ for j, img in enumerate(row):+ grid.paste(img, box=(j*w, i*h))+ return grid++ def lookup_score_from_fname(self,+ fname: str,+ metrics_output_file: str+ ) -> float:+ fname_basestr = os.path.splitext(fname)[0]+ with open(metrics_output_file, 'r') as f:+ for line in f:+ if fname_basestr in line:+ score = float(line.split('|')[5])+ return score+ raise ValueError(f"Could not find score for {fname} in {metrics_output_file}")++ def gather_file_basenames(self, directory: str):+ files = []+ for file in os.listdir(directory):+ if file.endswith(".png"):+ files.append(file)+ return files++ def read_file_prompt(self, fname:str) -> str:+ # Read prompt from image file metadata+ img = Image.open(fname)+ img.load()+ return img.info['prompt']++ def find_file_match(self, baseline_file: str, file_paths: List[str]):+ # Find a file in file_paths with matching metadata to baseline_file+ baseline_prompt = self.read_file_prompt(baseline_file)++ # Do not match empty prompts+ if baseline_prompt is None or baseline_prompt == "":+ return None++ # Find file match+ # Reorder test_file_names so that the file with matching name is first+ # This is an optimization because matching file names are more likely+ # to have matching metadata if they were generated with the same script+ basename = os.path.basename(baseline_file)+ file_path_basenames = [os.path.basename(f) for f in file_paths]+ if basename in file_path_basenames:+ match_index = file_path_basenames.index(basename)+ file_paths.insert(0, file_paths.pop(match_index))++ for f in file_paths:+ test_file_prompt = self.read_file_prompt(f)+ if baseline_prompt == test_file_prompt:+ return f\ No newline at end of filediff --git a/tests/conftest.py b/tests/conftest.pynew file mode 100644index 000000000..1a35880af--- /dev/null+++ b/tests/conftest.py@@ -0,0 +1,36 @@+import os+import pytest++# Command line arguments for pytest+def pytest_addoption(parser):+ parser.addoption('--output_dir', action="store", default='tests/inference/samples', help='Output directory for generated images')+ parser.addoption("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0", help="Specify the IP address to listen on (default: 127.0.0.1). If --listen is provided without an argument, it defaults to 0.0.0.0. (listens on all)")+ parser.addoption("--port", type=int, default=8188, help="Set the listen port.")++# This initializes args at the beginning of the test session[email protected](scope="session", autouse=True)+def args_pytest(pytestconfig):+ args = {}+ args['output_dir'] = pytestconfig.getoption('output_dir')+ args['listen'] = pytestconfig.getoption('listen')+ args['port'] = pytestconfig.getoption('port')++ os.makedirs(args['output_dir'], exist_ok=True)++ return args++def pytest_collection_modifyitems(items):+ # Modifies items so tests run in the correct order++ LAST_TESTS = ['test_quality']++ # Move the last items to the end+ last_items = []+ for test_name in LAST_TESTS:+ for item in items.copy():+ print(item.module.__name__, item)+ if item.module.__name__ == test_name:+ last_items.append(item)+ items.remove(item)++ items.extend(last_items)diff --git a/tests/inference/__init__.py b/tests/inference/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/inference/graphs/default_graph_sdxl1_0.json b/tests/inference/graphs/default_graph_sdxl1_0.jsonnew file mode 100644index 000000000..c06c6829c--- /dev/null+++ b/tests/inference/graphs/default_graph_sdxl1_0.json@@ -0,0 +1,144 @@+{+ "4": {+ "inputs": {+ "ckpt_name": "sd_xl_base_1.0.safetensors"+ },+ "class_type": "CheckpointLoaderSimple"+ },+ "5": {+ "inputs": {+ "width": 1024,+ "height": 1024,+ "batch_size": 1+ },+ "class_type": "EmptyLatentImage"+ },+ "6": {+ "inputs": {+ "text": "a photo of a cat",+ "clip": [+ "4",+ 1+ ]+ },+ "class_type": "CLIPTextEncode"+ },+ "10": {+ "inputs": {+ "add_noise": "enable",+ "noise_seed": 42,+ "steps": 20,+ "cfg": 7.5,+ "sampler_name": "euler",+ "scheduler": "normal",+ "start_at_step": 0,+ "end_at_step": 32,+ "return_with_leftover_noise": "enable",+ "model": [+ "4",+ 0+ ],+ "positive": [+ "6",+ 0+ ],+ "negative": [+ "15",+ 0+ ],+ "latent_image": [+ "5",+ 0+ ]+ },+ "class_type": "KSamplerAdvanced"+ },+ "12": {+ "inputs": {+ "samples": [+ "14",+ 0+ ],+ "vae": [+ "4",+ 2+ ]+ },+ "class_type": "VAEDecode"+ },+ "13": {+ "inputs": {+ "filename_prefix": "test_inference",+ "images": [+ "12",+ 0+ ]+ },+ "class_type": "SaveImage"+ },+ "14": {+ "inputs": {+ "add_noise": "disable",+ "noise_seed": 42,+ "steps": 20,+ "cfg": 7.5,+ "sampler_name": "euler",+ "scheduler": "normal",+ "start_at_step": 32,+ "end_at_step": 10000,+ "return_with_leftover_noise": "disable",+ "model": [+ "16",+ 0+ ],+ "positive": [+ "17",+ 0+ ],+ "negative": [+ "20",+ 0+ ],+ "latent_image": [+ "10",+ 0+ ]+ },+ "class_type": "KSamplerAdvanced"+ },+ "15": {+ "inputs": {+ "conditioning": [+ "6",+ 0+ ]+ },+ "class_type": "ConditioningZeroOut"+ },+ "16": {+ "inputs": {+ "ckpt_name": "sd_xl_refiner_1.0.safetensors"+ },+ "class_type": "CheckpointLoaderSimple"+ },+ "17": {+ "inputs": {+ "text": "a photo of a cat",+ "clip": [+ "16",+ 1+ ]+ },+ "class_type": "CLIPTextEncode"+ },+ "20": {+ "inputs": {+ "text": "",+ "clip": [+ "16",+ 1+ ]+ },+ "class_type": "CLIPTextEncode"+ }+ }\ No newline at end of filediff --git a/tests/inference/test_inference.py b/tests/inference/test_inference.pynew file mode 100644index 000000000..a96f94550--- /dev/null+++ b/tests/inference/test_inference.py@@ -0,0 +1,247 @@+from copy import deepcopy+from io import BytesIO+from urllib import request+import numpy+import os+from PIL import Image+import pytest+from pytest import fixture+import time+import torch+from typing import Union+import json+import subprocess+import websocket #NOTE: websocket-client (https://github.com/websocket-client/websocket-client)+import uuid+import urllib.request+import urllib.parse++# Currently causes an error when running pytest with built-in pytest args+# TODO: modify cli_args.py to not parse args on import+# We will hard-code sampler and scheduler lists for now+# from comfy.samplers import KSampler++"""+These tests generate and save images through a range of parameters+"""++class ComfyGraph:+ def __init__(self,+ graph: dict,+ sampler_nodes: list[str],+ ):+ self.graph = graph+ self.sampler_nodes = sampler_nodes++ def set_prompt(self, prompt, negative_prompt=None):+ # Sets the prompt for the sampler nodes (eg. base and refiner)+ for node in self.sampler_nodes:+ prompt_node = self.graph[node]['inputs']['positive'][0]+ self.graph[prompt_node]['inputs']['text'] = prompt+ if negative_prompt:+ negative_prompt_node = self.graph[node]['inputs']['negative'][0]+ self.graph[negative_prompt_node]['inputs']['text'] = negative_prompt++ def set_sampler_name(self, sampler_name:str, ):+ # sets the sampler name for the sampler nodes (eg. base and refiner)+ for node in self.sampler_nodes:+ self.graph[node]['inputs']['sampler_name'] = sampler_name++ def set_scheduler(self, scheduler:str):+ # sets the sampler name for the sampler nodes (eg. base and refiner)+ for node in self.sampler_nodes:+ self.graph[node]['inputs']['scheduler'] = scheduler++ def set_filename_prefix(self, prefix:str):+ # sets the filename prefix for the save nodes+ for node in self.graph:+ if self.graph[node]['class_type'] == 'SaveImage':+ self.graph[node]['inputs']['filename_prefix'] = prefix+++class ComfyClient:+ # From examples/websockets_api_example.py++ def connect(self,+ listen:str = '127.0.0.1',+ port:Union[str,int] = 8188,+ client_id: str = str(uuid.uuid4())+ ):+ self.client_id = client_id+ self.server_address = f"{listen}:{port}"+ ws = websocket.WebSocket()+ ws.connect("ws://{}/ws?clientId={}".format(self.server_address, self.client_id))+ self.ws = ws++ def queue_prompt(self, prompt):+ p = {"prompt": prompt, "client_id": self.client_id}+ data = json.dumps(p).encode('utf-8')+ req = urllib.request.Request("http://{}/prompt".format(self.server_address), data=data)+ return json.loads(urllib.request.urlopen(req).read())++ def get_image(self, filename, subfolder, folder_type):+ data = {"filename": filename, "subfolder": subfolder, "type": folder_type}+ url_values = urllib.parse.urlencode(data)+ with urllib.request.urlopen("http://{}/view?{}".format(self.server_address, url_values)) as response:+ return response.read()++ def get_history(self, prompt_id):+ with urllib.request.urlopen("http://{}/history/{}".format(self.server_address, prompt_id)) as response:+ return json.loads(response.read())++ def get_images(self, graph, save=True):+ prompt = graph+ if not save:+ # Replace save nodes with preview nodes+ prompt_str = json.dumps(prompt)+ prompt_str = prompt_str.replace('SaveImage', 'PreviewImage')+ prompt = json.loads(prompt_str)++ prompt_id = self.queue_prompt(prompt)['prompt_id']+ output_images = {}+ while True:+ out = self.ws.recv()+ if isinstance(out, str):+ message = json.loads(out)+ if message['type'] == 'executing':+ data = message['data']+ if data['node'] is None and data['prompt_id'] == prompt_id:+ break #Execution is done+ else:+ continue #previews are binary data++ history = self.get_history(prompt_id)[prompt_id]+ for o in history['outputs']:+ for node_id in history['outputs']:+ node_output = history['outputs'][node_id]+ if 'images' in node_output:+ images_output = []+ for image in node_output['images']:+ image_data = self.get_image(image['filename'], image['subfolder'], image['type'])+ images_output.append(image_data)+ output_images[node_id] = images_output++ return output_images++#+# Initialize graphs+#+default_graph_file = 'tests/inference/graphs/default_graph_sdxl1_0.json'+with open(default_graph_file, 'r') as file:+ default_graph = json.loads(file.read())+DEFAULT_COMFY_GRAPH = ComfyGraph(graph=default_graph, sampler_nodes=['10','14'])+DEFAULT_COMFY_GRAPH_ID = os.path.splitext(os.path.basename(default_graph_file))[0]++#+# Loop through these variables+#+comfy_graph_list = [DEFAULT_COMFY_GRAPH]+comfy_graph_ids = [DEFAULT_COMFY_GRAPH_ID]+prompt_list = [+ 'a painting of a cat',+]+#TODO use sampler and scheduler list from comfy.samplers.KSampler+# sampler_list = KSampler.SAMPLERS+# scheduler_list = KSampler.SCHEDULERS+# Hard coded sampler and scheduler lists for now+SCHEDULERS = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform"]+SAMPLERS = ["euler", "euler_ancestral", "heun", "dpm_2", "dpm_2_ancestral",+ "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu",+ "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddim", "uni_pc", "uni_pc_bh2"]+sampler_list = SAMPLERS+scheduler_list = SCHEDULERS[email protected]("sampler", sampler_list)[email protected]("scheduler", scheduler_list)[email protected]("prompt", prompt_list)+class TestInference:+ #+ # Initialize server and client+ #+ @fixture(scope="class", autouse=True)+ def _server(self, args_pytest):+ # Start server+ p = subprocess.Popen([+ 'python','main.py',+ '--output-directory', args_pytest["output_dir"],+ '--listen', args_pytest["listen"],+ '--port', str(args_pytest["port"]),+ ])+ yield+ p.kill()+ torch.cuda.empty_cache()++ def start_client(self, listen:str, port:int):+ # Start client+ comfy_client = ComfyClient()+ # Connect to server (with retries)+ n_tries = 5+ for i in range(n_tries):+ time.sleep(4)+ try:+ comfy_client.connect(listen=listen, port=port)+ except ConnectionRefusedError as e:+ print(e)+ print(f"({i+1}/{n_tries}) Retrying...")+ else:+ break+ return comfy_client++ #+ # Client and graph fixtures with server warmup+ #+ # Returns a "_client_graph", which is client-graph pair corresponding to an initialized server+ # The "graph" is the default graph+ @fixture(scope="class", params=comfy_graph_list, ids=comfy_graph_ids, autouse=True)+ def _client_graph(self, request, args_pytest, _server) -> (ComfyClient, ComfyGraph):+ comfy_graph = request.param++ # Start client+ comfy_client = self.start_client(args_pytest["listen"], args_pytest["port"])++ # Warm up pipeline+ comfy_client.get_images(graph=comfy_graph.graph, save=False)++ yield comfy_client, comfy_graph+ del comfy_client+ del comfy_graph+ torch.cuda.empty_cache()++ @fixture+ def client(self, _client_graph):+ client = _client_graph[0]+ yield client++ @fixture+ def comfy_graph(self, _client_graph):+ # avoid mutating the graph+ graph = deepcopy(_client_graph[1])+ yield graph++ def test_comfy(+ self,+ client,+ comfy_graph,+ sampler,+ scheduler,+ prompt,+ request+ ):+ test_info = request.node.name+ comfy_graph.set_filename_prefix(test_info)+ # Settings for comfy graph+ comfy_graph.set_sampler_name(sampler)+ comfy_graph.set_scheduler(scheduler)+ comfy_graph.set_prompt(prompt)++ # Generate+ images = client.get_images(comfy_graph.graph)++ assert len(images) != 0, "No images generated"+ # assert all images are not blank+ for images_output in images.values():+ for image_data in images_output:+ pil_image = Image.open(BytesIO(image_data))+ assert numpy.array(pil_image).any() != 0, "Image is blank"++