Skip to content

Industry leading face manipulation platform

Feat/halt on error (#862)

Henry Ruhs296eea8577a0
diff --git a/facefusion.ini b/facefusion.ini
index 96b006e6..b81c657a 100644
--- a/facefusion.ini
+++ b/facefusion.ini
@@ -115,3 +115,4 @@ system_memory_limit =
[misc]
log_level =
+halt_on_error =
diff --git a/facefusion/args.py b/facefusion/args.py
index 5538a4b5..7cccdaf9 100644
--- a/facefusion/args.py
+++ b/facefusion/args.py
@@ -129,6 +129,7 @@ def apply_args(args : Args, apply_state_item : ApplyStateItem) -> None:
apply_state_item('system_memory_limit', args.get('system_memory_limit'))
# misc
apply_state_item('log_level', args.get('log_level'))
+ apply_state_item('halt_on_error', args.get('halt_on_error'))
# jobs
apply_state_item('job_id', args.get('job_id'))
apply_state_item('job_status', args.get('job_status'))
diff --git a/facefusion/core.py b/facefusion/core.py
index 19ac0cdc..f2e980ef 100755
--- a/facefusion/core.py
+++ b/facefusion/core.py
@@ -183,7 +183,7 @@ def route_job_manager(args : Args) -> ErrorCode:
return 1
if state_manager.get_item('command') == 'job-submit-all':
- if job_manager.submit_jobs():
+ if job_manager.submit_jobs(state_manager.get_item('halt_on_error')):
logger.info(wording.get('job_all_submitted'), __name__)
return 0
logger.error(wording.get('job_all_not_submitted'), __name__)
@@ -197,7 +197,7 @@ def route_job_manager(args : Args) -> ErrorCode:
return 1
if state_manager.get_item('command') == 'job-delete-all':
- if job_manager.delete_jobs():
+ if job_manager.delete_jobs(state_manager.get_item('halt_on_error')):
logger.info(wording.get('job_all_deleted'), __name__)
return 0
logger.error(wording.get('job_all_not_deleted'), __name__)
@@ -250,7 +250,7 @@ def route_job_runner() -> ErrorCode:
if state_manager.get_item('command') == 'job-run-all':
logger.info(wording.get('running_jobs'), __name__)
- if job_runner.run_jobs(process_step):
+ if job_runner.run_jobs(process_step, state_manager.get_item('halt_on_error')):
logger.info(wording.get('processing_jobs_succeed'), __name__)
return 0
logger.info(wording.get('processing_jobs_failed'), __name__)
@@ -266,7 +266,7 @@ def route_job_runner() -> ErrorCode:
if state_manager.get_item('command') == 'job-retry-all':
logger.info(wording.get('retrying_jobs'), __name__)
- if job_runner.retry_jobs(process_step):
+ if job_runner.retry_jobs(process_step, state_manager.get_item('halt_on_error')):
logger.info(wording.get('processing_jobs_succeed'), __name__)
return 0
logger.info(wording.get('processing_jobs_failed'), __name__)
diff --git a/facefusion/jobs/job_manager.py b/facefusion/jobs/job_manager.py
index dedad90a..a503c869 100644
--- a/facefusion/jobs/job_manager.py
+++ b/facefusion/jobs/job_manager.py
@@ -48,14 +48,17 @@ def submit_job(job_id : str) -> bool:
return False
-def submit_jobs() -> bool:
+def submit_jobs(halt_on_error : bool) -> bool:
drafted_job_ids = find_job_ids('drafted')
+ has_error = False
if drafted_job_ids:
for job_id in drafted_job_ids:
if not submit_job(job_id):
- return False
- return True
+ has_error = True
+ if halt_on_error:
+ return False
+ return not has_error
return False
@@ -63,14 +66,17 @@ def delete_job(job_id : str) -> bool:
return delete_job_file(job_id)
-def delete_jobs() -> bool:
+def delete_jobs(halt_on_error : bool) -> bool:
job_ids = find_job_ids('drafted') + find_job_ids('queued') + find_job_ids('failed') + find_job_ids('completed')
+ has_error = False
if job_ids:
for job_id in job_ids:
if not delete_job(job_id):
- return False
- return True
+ has_error = True
+ if halt_on_error:
+ return False
+ return not has_error
return False
diff --git a/facefusion/jobs/job_runner.py b/facefusion/jobs/job_runner.py
index 33a2fafb..ab6f92a7 100644
--- a/facefusion/jobs/job_runner.py
+++ b/facefusion/jobs/job_runner.py
@@ -16,14 +16,17 @@ def run_job(job_id : str, process_step : ProcessStep) -> bool:
return False
-def run_jobs(process_step : ProcessStep) -> bool:
+def run_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool:
queued_job_ids = job_manager.find_job_ids('queued')
+ has_error = False
if queued_job_ids:
for job_id in queued_job_ids:
if not run_job(job_id, process_step):
- return False
- return True
+ has_error = True
+ if halt_on_error:
+ return False
+ return not has_error
return False
@@ -35,14 +38,17 @@ def retry_job(job_id : str, process_step : ProcessStep) -> bool:
return False
-def retry_jobs(process_step : ProcessStep) -> bool:
+def retry_jobs(process_step : ProcessStep, halt_on_error : bool) -> bool:
failed_job_ids = job_manager.find_job_ids('failed')
+ has_error = False
if failed_job_ids:
for job_id in failed_job_ids:
if not retry_job(job_id, process_step):
- return False
- return True
+ has_error = True
+ if halt_on_error:
+ return False
+ return not has_error
return False
diff --git a/facefusion/program.py b/facefusion/program.py
index 41e220ff..06672043 100755
--- a/facefusion/program.py
+++ b/facefusion/program.py
@@ -230,7 +230,7 @@ def create_memory_program() -> ArgumentParser:
return program
-def create_misc_program() -> ArgumentParser:
+def create_log_level_program() -> ArgumentParser:
program = ArgumentParser(add_help = False)
group_misc = program.add_argument_group('misc')
group_misc.add_argument('--log-level', help = wording.get('help.log_level'), default = config.get_str_value('misc.log_level', 'info'), choices = facefusion.choices.log_levels)
@@ -238,6 +238,14 @@ def create_misc_program() -> ArgumentParser:
return program
+def create_halt_on_error_program() -> ArgumentParser:
+ program = ArgumentParser(add_help = False)
+ group_misc = program.add_argument_group('misc')
+ group_misc.add_argument('--halt-on-error', help = wording.get('help.halt_on_error'), action = 'store_true', default = config.get_bool_value('misc.halt_on_error'))
+ job_store.register_job_keys([ 'halt_on_error' ])
+ return program
+
+
def create_job_id_program() -> ArgumentParser:
program = ArgumentParser(add_help = False)
program.add_argument('job_id', help = wording.get('help.job_id'))
@@ -262,7 +270,7 @@ def collect_step_program() -> ArgumentParser:
def collect_job_program() -> ArgumentParser:
- return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_misc_program() ], add_help = False)
+ return ArgumentParser(parents= [ create_execution_program(), create_download_providers_program(), create_memory_program(), create_log_level_program() ], add_help = False)
def create_program() -> ArgumentParser:
@@ -274,23 +282,23 @@ def create_program() -> ArgumentParser:
sub_program.add_parser('run', help = wording.get('help.run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_uis_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('headless-run', help = wording.get('help.headless_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('batch-run', help = wording.get('help.batch_run'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), create_source_pattern_program(), create_target_pattern_program(), create_output_pattern_program(), collect_step_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('force-download', help = wording.get('help.force_download'), parents = [ create_download_providers_program(), create_download_scope_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
# job manager
- sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_misc_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-list', help = wording.get('help.job_list'), parents = [ create_job_status_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-create', help = wording.get('help.job_create'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-submit', help = wording.get('help.job_submit'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-submit-all', help = wording.get('help.job_submit_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-delete', help = wording.get('help.job_delete'), parents = [ create_job_id_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-delete-all', help = wording.get('help.job_delete_all'), parents = [ create_jobs_path_program(), create_log_level_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-add-step', help = wording.get('help.job_add_step'), parents = [ create_job_id_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-remix-step', help = wording.get('help.job_remix_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-insert-step', help = wording.get('help.job_insert_step'), parents = [ create_job_id_program(), create_step_index_program(), create_config_path_program(), create_jobs_path_program(), create_source_paths_program(), create_target_path_program(), create_output_path_program(), collect_step_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-remove-step', help = wording.get('help.job_remove_step'), parents = [ create_job_id_program(), create_step_index_program(), create_jobs_path_program(), create_log_level_program() ], formatter_class = create_help_formatter_large)
# job runner
sub_program.add_parser('job-run', help = wording.get('help.job_run'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-run-all', help = wording.get('help.job_run_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)
sub_program.add_parser('job-retry', help = wording.get('help.job_retry'), parents = [ create_job_id_program(), create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
- sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program() ], formatter_class = create_help_formatter_large)
+ sub_program.add_parser('job-retry-all', help = wording.get('help.job_retry_all'), parents = [ create_config_path_program(), create_temp_path_program(), create_jobs_path_program(), collect_job_program(), create_halt_on_error_program() ], formatter_class = create_help_formatter_large)
return ArgumentParser(parents = [ program ], formatter_class = create_help_formatter_small)
diff --git a/facefusion/typing.py b/facefusion/typing.py
index 10ef6ec3..03e54835 100755
--- a/facefusion/typing.py
+++ b/facefusion/typing.py
@@ -283,6 +283,7 @@ StateKey = Literal\
'video_memory_strategy',
'system_memory_limit',
'log_level',
+ 'halt_on_error',
'job_id',
'job_status',
'step_index'
@@ -347,6 +348,7 @@ State = TypedDict('State',
'video_memory_strategy' : VideoMemoryStrategy,
'system_memory_limit' : int,
'log_level' : LogLevel,
+ 'halt_on_error' : bool,
'job_id' : str,
'job_status' : JobStatus,
'step_index' : int
diff --git a/facefusion/uis/components/job_manager.py b/facefusion/uis/components/job_manager.py
index 893e433a..727aaf98 100644
--- a/facefusion/uis/components/job_manager.py
+++ b/facefusion/uis/components/job_manager.py
@@ -89,6 +89,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
if is_directory(step_args.get('output_path')):
step_args['output_path'] = suggest_output_path(step_args.get('output_path'), state_manager.get_item('target_path'))
+
if job_action == 'job-create':
if created_job_id and job_manager.create_job(created_job_id):
updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]
@@ -97,6 +98,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
return gradio.Dropdown(value = 'job-add-step'), gradio.Textbox(visible = False), gradio.Dropdown(value = created_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown()
else:
logger.error(wording.get('job_not_created').format(job_id = created_job_id), __name__)
+
if job_action == 'job-submit':
if selected_job_id and job_manager.submit_job(selected_job_id):
updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]
@@ -105,6 +107,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown()
else:
logger.error(wording.get('job_not_submitted').format(job_id = selected_job_id), __name__)
+
if job_action == 'job-delete':
if selected_job_id and job_manager.delete_job(selected_job_id):
updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ]
@@ -113,6 +116,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
return gradio.Dropdown(), gradio.Textbox(), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True), gradio.Dropdown()
else:
logger.error(wording.get('job_not_deleted').format(job_id = selected_job_id), __name__)
+
if job_action == 'job-add-step':
if selected_job_id and job_manager.add_step(selected_job_id, step_args):
state_manager.set_item('output_path', output_path)
@@ -121,6 +125,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
else:
state_manager.set_item('output_path', output_path)
logger.error(wording.get('job_step_not_added').format(job_id = selected_job_id), __name__)
+
if job_action == 'job-remix-step':
if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remix_step(selected_job_id, selected_step_index, step_args):
updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]
@@ -131,6 +136,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
else:
state_manager.set_item('output_path', output_path)
logger.error(wording.get('job_remix_step_not_added').format(job_id = selected_job_id, step_index = selected_step_index), __name__)
+
if job_action == 'job-insert-step':
if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.insert_step(selected_job_id, selected_step_index, step_args):
updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]
@@ -141,6 +147,7 @@ def apply(job_action : JobManagerAction, created_job_id : str, selected_job_id :
else:
state_manager.set_item('output_path', output_path)
logger.error(wording.get('job_step_not_inserted').format(job_id = selected_job_id, step_index = selected_step_index), __name__)
+
if job_action == 'job-remove-step':
if selected_job_id and job_manager.has_step(selected_job_id, selected_step_index) and job_manager.remove_step(selected_job_id, selected_step_index):
updated_step_choices = get_step_choices(selected_job_id) or [ 'none' ] #type:ignore[list-item]
@@ -160,16 +167,19 @@ def get_step_choices(job_id : str) -> List[int]:
def update(job_action : JobManagerAction, selected_job_id : str) -> Tuple[gradio.Textbox, gradio.Dropdown, gradio.Dropdown]:
if job_action == 'job-create':
return gradio.Textbox(value = None, visible = True), gradio.Dropdown(visible = False), gradio.Dropdown(visible = False)
+
if job_action == 'job-delete':
updated_job_ids = job_manager.find_job_ids('drafted') + job_manager.find_job_ids('queued') + job_manager.find_job_ids('failed') + job_manager.find_job_ids('completed') or [ 'none' ]
updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)
return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False)
+
if job_action in [ 'job-submit', 'job-add-step' ]:
updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]
updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)
return gradio.Textbox(visible = False), gradio.Dropdown(value = updated_job_id, choices = updated_job_ids, visible = True), gradio.Dropdown(visible = False)
+
if job_action in [ 'job-remix-step', 'job-insert-step', 'job-remove-step' ]:
updated_job_ids = job_manager.find_job_ids('drafted') or [ 'none' ]
updated_job_id = selected_job_id if selected_job_id in updated_job_ids else get_last(updated_job_ids)
diff --git a/facefusion/uis/components/job_runner.py b/facefusion/uis/components/job_runner.py
index 540a1025..90bccb20 100644
--- a/facefusion/uis/components/job_runner.py
+++ b/facefusion/uis/components/job_runner.py
@@ -95,12 +95,15 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, grad
updated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ]
return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids)
+
if job_action == 'job-run-all':
logger.info(wording.get('running_jobs'), __name__)
- if job_runner.run_jobs(process_step):
+ halt_on_error = False
+ if job_runner.run_jobs(process_step, halt_on_error):
logger.info(wording.get('processing_jobs_succeed'), __name__)
else:
logger.info(wording.get('processing_jobs_failed'), __name__)
+
if job_action == 'job-retry':
logger.info(wording.get('retrying_job').format(job_id = job_id), __name__)
if job_id and job_runner.retry_job(job_id, process_step):
@@ -110,9 +113,11 @@ def run(job_action : JobRunnerAction, job_id : str) -> Tuple[gradio.Button, grad
updated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ]
return gradio.Button(visible = True), gradio.Button(visible = False), gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids)
+
if job_action == 'job-retry-all':
logger.info(wording.get('retrying_jobs'), __name__)
- if job_runner.retry_jobs(process_step):
+ halt_on_error = False
+ if job_runner.retry_jobs(process_step, halt_on_error):
logger.info(wording.get('processing_jobs_succeed'), __name__)
else:
logger.info(wording.get('processing_jobs_failed'), __name__)
@@ -129,6 +134,7 @@ def update_job_action(job_action : JobRunnerAction) -> gradio.Dropdown:
updated_job_ids = job_manager.find_job_ids('queued') or [ 'none' ]
return gradio.Dropdown(value = get_last(updated_job_ids), choices = updated_job_ids, visible = True)
+
if job_action == 'job-retry':
updated_job_ids = job_manager.find_job_ids('failed') or [ 'none' ]
diff --git a/facefusion/wording.py b/facefusion/wording.py
index 93a8572a..8cc3c58f 100755
--- a/facefusion/wording.py
+++ b/facefusion/wording.py
@@ -200,6 +200,7 @@ WORDING : Dict[str, Any] =\
'system_memory_limit': 'limit the available RAM that can be used while processing',
# misc
'log_level': 'adjust the message severity displayed in the terminal',
+ 'halt_on_error': 'halt the program once an error occurred',
# run
'run': 'run the program',
'headless_run': 'run the program in headless mode',
diff --git a/tests/test_cli_job_manager.py b/tests/test_cli_job_manager.py
index fac789c8..a20a2728 100644
--- a/tests/test_cli_job_manager.py
+++ b/tests/test_cli_job_manager.py
@@ -61,7 +61,7 @@ def test_job_submit() -> None:
def test_submit_all() -> None:
- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
@@ -79,7 +79,7 @@ def test_submit_all() -> None:
commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-submit-all-2', '--jobs-path', get_test_jobs_directory(), '-s', get_test_example_file('source.jpg'), '-t', get_test_example_file('target-240p.jpg'), '-o', get_test_output_file('test-job-remix-step.jpg') ]
subprocess.run(commands)
- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 0
assert is_test_job_file('test-job-submit-all-1.json', 'queued') is True
@@ -103,7 +103,7 @@ def test_job_delete() -> None:
def test_job_delete_all() -> None:
- commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
@@ -113,7 +113,7 @@ def test_job_delete_all() -> None:
commands = [ sys.executable, 'facefusion.py', 'job-create', 'test-job-delete-all-2', '--jobs-path', get_test_jobs_directory() ]
subprocess.run(commands)
- commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-delete-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 0
assert is_test_job_file('test-job-delete-all-1.json', 'drafted') is False
diff --git a/tests/test_cli_job_runner.py b/tests/test_cli_job_runner.py
index 89dbad54..23d23cba 100644
--- a/tests/test_cli_job_runner.py
+++ b/tests/test_cli_job_runner.py
@@ -51,7 +51,7 @@ def test_job_run() -> None:
def test_job_run_all() -> None:
- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
@@ -70,14 +70,14 @@ def test_job_run_all() -> None:
commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-run-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-run-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ]
subprocess.run(commands)
- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
- commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-submit-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
subprocess.run(commands)
- commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-run-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 0
assert subprocess.run(commands).returncode == 1
@@ -111,7 +111,7 @@ def test_job_retry() -> None:
def test_job_retry_all() -> None:
- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
@@ -130,7 +130,7 @@ def test_job_retry_all() -> None:
commands = [ sys.executable, 'facefusion.py', 'job-add-step', 'test-job-retry-all-2', '--jobs-path', get_test_jobs_directory(), '--processors', 'face_debugger', '-t', get_test_example_file('target-240p.mp4'), '-o', get_test_output_file('test-job-retry-all-2.mp4'), '--trim-frame-start', '0', '--trim-frame-end', '1' ]
subprocess.run(commands)
- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 1
@@ -139,7 +139,7 @@ def test_job_retry_all() -> None:
move_job_file('test-job-retry-all-1', 'failed')
move_job_file('test-job-retry-all-2', 'failed')
- commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory() ]
+ commands = [ sys.executable, 'facefusion.py', 'job-retry-all', '--jobs-path', get_test_jobs_directory(), '--halt-on-error' ]
assert subprocess.run(commands).returncode == 0
assert subprocess.run(commands).returncode == 1
diff --git a/tests/test_job_manager.py b/tests/test_job_manager.py
index 3ee6c0d1..3de99ec0 100644
--- a/tests/test_job_manager.py
+++ b/tests/test_job_manager.py
@@ -63,19 +63,20 @@ def test_submit_jobs() -> None:
'target_path': 'target-2.jpg',
'output_path': 'output-2.jpg'
}
+ halt_on_error = True
- assert submit_jobs() is False
+ assert submit_jobs(halt_on_error) is False
create_job('job-test-submit-jobs-1')
create_job('job-test-submit-jobs-2')
- assert submit_jobs() is False
+ assert submit_jobs(halt_on_error) is False
add_step('job-test-submit-jobs-1', args_1)
add_step('job-test-submit-jobs-2', args_2)
- assert submit_jobs() is True
- assert submit_jobs() is False
+ assert submit_jobs(halt_on_error) is True
+ assert submit_jobs(halt_on_error) is False
def test_delete_job() -> None:
@@ -88,12 +89,14 @@ def test_delete_job() -> None:
def test_delete_jobs() -> None:
- assert delete_jobs() is False
+ halt_on_error = True
+
+ assert delete_jobs(halt_on_error) is False
create_job('job-test-delete-jobs-1')
create_job('job-test-delete-jobs-2')
- assert delete_jobs() is True
+ assert delete_jobs(halt_on_error) is True
@pytest.mark.skip()
diff --git a/tests/test_job_runner.py b/tests/test_job_runner.py
index a51968d2..7c14207c 100644
--- a/tests/test_job_runner.py
+++ b/tests/test_job_runner.py
@@ -85,8 +85,9 @@ def test_run_jobs() -> None:
'target_path': get_test_example_file('target-240p.jpg'),
'output_path': get_test_output_file('output-1.jpg')
}
+ halt_on_error = True
- assert run_jobs(process_step) is False
+ assert run_jobs(process_step, halt_on_error) is False
create_job('job-test-run-jobs-1')
create_job('job-test-run-jobs-2')
@@ -95,11 +96,11 @@ def test_run_jobs() -> None:
add_step('job-test-run-jobs-2', args_2)
add_step('job-test-run-jobs-3', args_3)
- assert run_jobs(process_step) is False
+ assert run_jobs(process_step, halt_on_error) is False
- submit_jobs()
+ submit_jobs(halt_on_error)
- assert run_jobs(process_step) is True
+ assert run_jobs(process_step, halt_on_error) is True
@pytest.mark.skip()