Industry leading face manipulation platform
Feat/halt on error (#862)
Henry Ruhs296eea8577a0
diff --git a/facefusion.ini b/facefusion.iniindex 96b006e6..b81c657a 100644--- a/facefusion.ini+++ b/facefusion.ini@@ -115,3 +115,4 @@ system_memory_limit =[misc]log_level =+halt_on_error =diff --git a/facefusion/args.py b/facefusion/args.pyindex 5538a4b5..7cccdaf9 100644--- a/facefusion/args.py+++ b/facefusion/args.py@@ -129,6 +129,7 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:apply_state_item('system_memory_limit', args.get('system_memory_limit'))# miscapply_state_item('log_level', args.get('log_level'))+ apply_state_item('halt_on_error', args.get('halt_on_error'))# jobsapply_state_item('job_id', args.get('job_id'))apply_state_item('job_status', args.get('job_status'))diff --git a/facefusion/core.py b/facefusion/core.pyindex 19ac0cdc..f2e980ef 100755--- a/facefusion/core.py+++ b/facefusion/core.py@@ -183,7 +183,7 @@ def route_job_manager(args : Args) -> ErrorCode:return 1if state_manager.get_item('command') == 'job-submit-all':- if job_manager.submit_jobs():+ if job_manager.submit_jobs(state_manager.get_item('halt_on_error')):logger.info(wording.get('job_all_submitted'), __name__)return 0logger.error(wording.get('job_all_not_submitted'), __name__)@@ -197,7 +197,7 @@ def route_job_manager(args : Args) -> ErrorCode:return 1if state_manager.get_item('command') == 'job-delete-all':- if job_manager.delete_jobs():+ if job_manager.delete_jobs(state_manager.get_item('halt_on_error')):logger.info(wording.get('job_all_deleted'), __name__)return 0logger.error(wording.get('job_all_not_deleted'), __name__)@@ -250,7 +250,7 @@ def route_job_runner() -> ErrorCode:if state_manager.get_item('command') == 'job-run-all':logger.info(wording.get('running_jobs'), __name__)- if job_runner.run_jobs(process_step):+ if job_runner.run_jobs(process_step, state_manager.get_item('halt_on_error')):logger.info(wording.get('processing_jobs_succeed'), __name__)return 0logger.info(wording.get('processing_jobs_failed'), __name__)@@ -266,7 +266,7 @@ def route_job_runner() -> ErrorCode:if state_manager.get_item('command') == 'job-retry-all':logger.info(wording.get('retrying_jobs'), __name__)- if job_runner.retry_jobs(process_step):+ if job_runner.retry_jobs(process_step, state_manager.get_item('halt_on_error')):logger.info(wording.get('processing_jobs_succeed'), __name__)return 0logger.info(wording.get('processing_jobs_failed'), __name__)diff --git a/facefusion/jobs/job_manager.py b/facefusion/jobs/job_manager.pyindex dedad90a..a503c869 100644--- a/facefusion/jobs/job_manager.py+++ b/facefusion/jobs/job_manager.py@@ -48,14 +48,17 @@ def submit_job(job_id : str) -> bool:return False-def submit_jobs() -> bool:+def submit_jobs(halt_on_error : bool) -> bool:drafted_job_ids = find_job_ids('drafted')+ has_error = Falseif drafted_job_ids:for job_id in drafted_job_ids:if not submit_job(job_id):- return False- return True+ has_error = True+ if halt_on_error:+ return False+ return not has_errorreturn False@@ -63,14 +66,17 @@ def delete_job(job_id : str) -> bool:return delete_job_file(job_id)-def delete_jobs() -> bool:+def delete_jobs(halt_on_error : bool) -> bool:job_ids = find_job_ids('drafted') + find_job_ids('queued') + find_job_ids('failed') + find_job_ids('completed')+ has_error = Falseif job_ids:for job_id in job_ids:if not delete_job(job_id):- return False- return True+ has_error = True+ if halt_on_error:+ return False+ return not has_errorreturn Falsediff --git a/facefusion/jobs/job_runner.py b/facefusion/jobs/job_runner.pyindex 33a2fafb..ab6f92a7 100644--- a/facefusion/jobs/job_runner.py+++ b/facefusion/jobs/job_runner.py@@ -16,14 +16,17 @@ def run_job(job_id : str, process_step : ProcessStep) -> bool:return False-def run_jobs(process_step : ProcessStep) -> bool:+def run_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool:queued_job_ids = job_manager.find_job_ids('queued')+ has_error = Falseif queued_job_ids:for job_id in queued_job_ids:if not run_job(job_id, process_step):- return False- return True+ has_error = True+ if halt_on_error:+ return False+ return not has_errorreturn False@@ -35,14 +38,17 @@ def retry_job(job_id : str, process_step : ProcessStep) -> bool:return False-def retry_jobs(process_step : ProcessStep) -> bool:+def retry_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool:failed_job_ids = job_manager.find_job_ids('failed')+ has_error = Falseif failed_job_ids:for job_id in failed_job_ids:if not retry_job(job_id, process_step):- return False- return True+ has_error = True+ if halt_on_error:+ return False+ return not has_errorreturn Falsediff --git a/facefusion/program.py b/facefusion/program.pyindex 41e220ff..06672043 100755--- a/facefusion/program.py+++ b/facefusion/program.py@@ -230,7 +230,7 @@ def create_memory_program() -> ArgumentParser:return program-def create_misc_program() -> ArgumentParser:+def create_log_level_program() -> ArgumentParser:program = ArgumentParser(add_help = False)group_misc = program.add_argument_group('misc')group_misc.add_argument('--log-level', help = wording.get('help.log_level'), default = config.get_str_value('misc.log_level', 'info'), choices = facefusion.choices.log_levels)@@ -238,6 +238,14 @@ def create_misc_program() -> ArgumentParser:return program+def create_halt_on_error_program() -> ArgumentParser:+ program = ArgumentParser(add_help = False)+ group_misc = program.add_argument_group('misc')+ group_misc.add_argument('--halt-on-error', help = wording.get('help.halt_on_error'), action = 'store_true', default = config.get_bool_value('misc.halt_on_error'))+ job_store.register_job_keys([ 'halt_on_error' ])+ return program++def create_job_id_program() -> ArgumentParser:program = ArgumentParser(add_help = False)program.add_argument('job_id', help = wording.get('help.job_id'))@@ -262,7 +270,7 @@ def collect_step_program() -> ArgumentParser:def collect_job_program() -> ArgumentParser:- return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_misc_program() ], add_help = False)+ return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_log_level_program() ], add_help = False)def create_program() -> ArgumentParser:@@ -274,23 +282,23 @@ def create_program() -> ArgumentParser:sub_program.add_parser('run', help = wording.get('help.run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_uis_program(), collect_job_program() ], formatter_class = create_help_formatter_large)sub_program.add_parser('headless-run', help = wording.get('help.headless_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)sub_program.add_parser('batch-run', help = wording.get('help.batch_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_pattern_program(), create_target_pattern_program(), create_output_pattern_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_misc_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)# job manager- sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)# job runnersub_program.add_parser('job-run', help = wording.get('help.job_run'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)sub_program.add_parser('job-retry', help = wording.get('help.job_retry'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)- sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)+ sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)return ArgumentParser(parents = [ program ], formatter_class = create_help_formatter_small)diff --git a/facefusion/typing.py b/facefusion/typing.pyindex 10ef6ec3..03e54835 100755--- a/facefusion/typing.py+++ b/facefusion/typing.py@@ -283,6 +283,7 @@ StateKey = Literal\'video_memory_strategy','system_memory_limit','log_level',+ 'halt_on_error','job_id','job_status','step_index'@@ -347,6 +348,7 @@ State = TypedDict('State','video_memory_strategy' : VideoMemoryStrategy,'system_memory_limit' : int,'log_level' : LogLevel,+ 'halt_on_error' : bool,'job_id' : str,'job_status' : JobStatus,'step_index' : intdiff --git a/facefusion/uis/components/job_manager.py b/facefusion/uis/components/job_manager.pyindex 893e433a..727aaf98 100644--- a/facefusion/uis/components/job_manager.py+++ b/facefusion/uis/components/job_manager.py@@ -89,6 +89,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :if is_directory(step_args.get('output_path')):step_args['output_path'] = suggest_output_path(step_args.get('output_path'), state_manager.get_item('target_path'))+if job_action == 'job-create':if created_job_id and job_manager.create_job(created_job_id):updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]@@ -97,6 +98,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :return gradio.Dropdown(value = 'job-add-step'), gradio.Textbox(visible = False), gradio.Dropdown(value = created_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown()else:logger.error(wording.get('job_not_created').format(job_id = created_job_id), __name__)+if job_action == 'job-submit':if selected_job_id and job_manager.submit_job(selected_job_id):updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]@@ -105,6 +107,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown()else:logger.error(wording.get('job_not_submitted').format(job_id = selected_job_id), __name__)+if job_action == 'job-delete':if selected_job_id and job_manager.delete_job(selected_job_id):updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ]@@ -113,6 +116,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown()else:logger.error(wording.get('job_not_deleted').format(job_id = selected_job_id), __name__)+if job_action == 'job-add-step':if selected_job_id and job_manager.add_step(selected_job_id, step_args):state_manager.set_item('output_path', output_path)@@ -121,6 +125,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :else:state_manager.set_item('output_path', output_path)logger.error(wording.get('job_step_not_added').format(job_id = selected_job_id), __name__)+if job_action == 'job-remix-step':if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remix_step(selected_job_id, selected_step_index, step_args):updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]@@ -131,6 +136,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :else:state_manager.set_item('output_path', output_path)logger.error(wording.get('job_remix_step_not_added').format(job_id = selected_job_id, step_index = selected_step_index), __name__)+if job_action == 'job-insert-step':if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.insert_step(selected_job_id, selected_step_index, step_args):updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]@@ -141,6 +147,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :else:state_manager.set_item('output_path', output_path)logger.error(wording.get('job_step_not_inserted').format(job_id = selected_job_id, step_index = selected_step_index), __name__)+if job_action == 'job-remove-step':if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remove_step(selected_job_id, selected_step_index):updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]@@ -160,16 +167,19 @@ def get_step_choices(job_id : str) -> List[int]:def update(job_action : JobManagerAction, selected_job_id : str) -> Tuple[gradio.Textbox, gradio.Dropdown, gradio.Dropdown]:if job_action == 'job-create':return gradio.Textbox(value = None, visible = True), gradio.Dropdown(visible = False), gradio.Dropdown(visible = False)+if job_action == 'job-delete':updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ]updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False)+if job_action in [ 'job-submit', 'job-add-step' ]:updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False)+if job_action in [ 'job-remix-step', 'job-insert-step', 'job-remove-step' ]:updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)diff --git a/facefusion/uis/components/job_runner.py b/facefusion/uis/components/job_runner.pyindex 540a1025..90bccb20 100644--- a/facefusion/uis/components/job_runner.py+++ b/facefusion/uis/components/job_runner.py@@ -95,12 +95,15 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, gradupdated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ]return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids)+if job_action == 'job-run-all':logger.info(wording.get('running_jobs'), __name__)- if job_runner.run_jobs(process_step):+ halt_on_error = False+ if job_runner.run_jobs(process_step, halt_on_error):logger.info(wording.get('processing_jobs_succeed'), __name__)else:logger.info(wording.get('processing_jobs_failed'), __name__)+if job_action == 'job-retry':logger.info(wording.get('retrying_job').format(job_id = job_id), __name__)if job_id and job_runner.retry_job(job_id, process_step):@@ -110,9 +113,11 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, gradupdated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ]return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids)+if job_action == 'job-retry-all':logger.info(wording.get('retrying_jobs'), __name__)- if job_runner.retry_jobs(process_step):+ halt_on_error = False+ if job_runner.retry_jobs(process_step, halt_on_error):logger.info(wording.get('processing_jobs_succeed'), __name__)else:logger.info(wording.get('processing_jobs_failed'), __name__)@@ -129,6 +134,7 @@ def update_job_action(job_action : JobRunnerAction) -> gradio.Dropdown:updated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ]return gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True)+if job_action == 'job-retry':updated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ]diff --git a/facefusion/wording.py b/facefusion/wording.pyindex 93a8572a..8cc3c58f 100755--- a/facefusion/wording.py+++ b/facefusion/wording.py@@ -200,6 +200,7 @@ WORDING : Dict[str, Any] =\'system_memory_limit': 'limit the available RAM that can be used while processing',# misc'log_level': 'adjust the message severity displayed in the terminal',+ 'halt_on_error': 'halt the program once an error occurred',# run'run': 'run the program','headless_run': 'run the program in headless mode',diff --git a/tests/test_cli_job_manager.py b/tests/test_cli_job_manager.pyindex fac789c8..a20a2728 100644--- a/tests/test_cli_job_manager.py+++ b/tests/test_cli_job_manager.py@@ -61,7 +61,7 @@ def test_job_submit() -> None:def test_submit_all() -> None:- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1@@ -79,7 +79,7 @@ def test_submit_all() -> None:commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-submit-all-2', '--jobs-path', get_test_jobs_directory(), '-s', get_test_example_file('source.jpg'), '-t', get_test_example_file('target-240p.jpg'), '-o', get_test_output_file('test-job-remix-step.jpg') ]subprocess.run(commands)- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 0assert is_test_job_file('test-job-submit-all-1.json', 'queued') is True@@ -103,7 +103,7 @@ def test_job_delete() -> None:def test_job_delete_all() -> None:- commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1@@ -113,7 +113,7 @@ def test_job_delete_all() -> None:commands = [ sys.executable, 'facefusion.py', 'job-create', 'test-job-delete-all-2', '--jobs-path', get_test_jobs_directory() ]subprocess.run(commands)- commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 0assert is_test_job_file('test-job-delete-all-1.json', 'drafted') is Falsediff --git a/tests/test_cli_job_runner.py b/tests/test_cli_job_runner.pyindex 89dbad54..23d23cba 100644--- a/tests/test_cli_job_runner.py+++ b/tests/test_cli_job_runner.py@@ -51,7 +51,7 @@ def test_job_run() -> None:def test_job_run_all() -> None:- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1@@ -70,14 +70,14 @@ def test_job_run_all() -> None:commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-run-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-run-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ]subprocess.run(commands)- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]subprocess.run(commands)- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 0assert subprocess.run(commands).returncode == 1@@ -111,7 +111,7 @@ def test_job_retry() -> None:def test_job_retry_all() -> None:- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1@@ -130,7 +130,7 @@ def test_job_retry_all() -> None:commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-retry-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-retry-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ]subprocess.run(commands)- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 1@@ -139,7 +139,7 @@ def test_job_retry_all() -> None:move_job_file('test-job-retry-all-1', 'failed')move_job_file('test-job-retry-all-2', 'failed')- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]assert subprocess.run(commands).returncode == 0assert subprocess.run(commands).returncode == 1diff --git a/tests/test_job_manager.py b/tests/test_job_manager.pyindex 3ee6c0d1..3de99ec0 100644--- a/tests/test_job_manager.py+++ b/tests/test_job_manager.py@@ -63,19 +63,20 @@ def test_submit_jobs() -> None:'target_path': 'target-2.jpg','output_path': 'output-2.jpg'}+ halt_on_error = True- assert submit_jobs() is False+ assert submit_jobs(halt_on_error) is Falsecreate_job('job-test-submit-jobs-1')create_job('job-test-submit-jobs-2')- assert submit_jobs() is False+ assert submit_jobs(halt_on_error) is Falseadd_step('job-test-submit-jobs-1', args_1)add_step('job-test-submit-jobs-2', args_2)- assert submit_jobs() is True- assert submit_jobs() is False+ assert submit_jobs(halt_on_error) is True+ assert submit_jobs(halt_on_error) is Falsedef test_delete_job() -> None:@@ -88,12 +89,14 @@ def test_delete_job() -> None:def test_delete_jobs() -> None:- assert delete_jobs() is False+ halt_on_error = True++ assert delete_jobs(halt_on_error) is Falsecreate_job('job-test-delete-jobs-1')create_job('job-test-delete-jobs-2')- assert delete_jobs() is True+ assert delete_jobs(halt_on_error) is True@pytest.mark.skip()diff --git a/tests/test_job_runner.py b/tests/test_job_runner.pyindex a51968d2..7c14207c 100644--- a/tests/test_job_runner.py+++ b/tests/test_job_runner.py@@ -85,8 +85,9 @@ def test_run_jobs() -> None:'target_path': get_test_example_file('target-240p.jpg'),'output_path': get_test_output_file('output-1.jpg')}+ halt_on_error = True- assert run_jobs(process_step) is False+ assert run_jobs(process_step, halt_on_error) is Falsecreate_job('job-test-run-jobs-1')create_job('job-test-run-jobs-2')@@ -95,11 +96,11 @@ def test_run_jobs() -> None:add_step('job-test-run-jobs-2', args_2)add_step('job-test-run-jobs-3', args_3)- assert run_jobs(process_step) is False+ assert run_jobs(process_step, halt_on_error) is False- submit_jobs()+ submit_jobs(halt_on_error)- assert run_jobs(process_step) is True+ assert run_jobs(process_step, halt_on_error) is True@pytest.mark.skip()