diff --git a/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py b/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py index 5e5ba45..bd9b396 100644 --- a/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py +++ b/workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py @@ -92,6 +92,7 @@ def parse_args(): parser.add_argument('--pause-between-steps', type=int, help='pause between steps', required=False, default=1) parser.add_argument('--face-detailer-enabled', help='face detailer enabled', required=False, default=False, action=argparse.BooleanOptionalAction) parser.add_argument('--upscaler-enabled', help='upscale enabled', required=False, default=False, action=argparse.BooleanOptionalAction) + parser.add_argument('--lipsync-enabled', help='lipsync enabled', required=False, default=False, action=argparse.BooleanOptionalAction) parser.add_argument('--strength', type=float, help='strength', required=False, default=1.0) return parser.parse_args() @@ -133,6 +134,7 @@ def main(): positive_prompt = None negative_prompt = None workflow_filename = None + enable_lipsync = args.lipsync_enabled if pipeline_type == PipelineType.IPA: workflow_filename = MAIN_IPA_WORKFLOW if positive_prompt_filename: @@ -142,7 +144,8 @@ def main(): validate_style_name(style) print("running pipeline" + str(pipeline_type)) # ensure_pipeline_input_present() - prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename) + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, + enable_lipsync=enable_lipsync) enqueue_prompt_and_wait(prompt) output_index = output_index + 1 left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" @@ -151,7 +154,8 @@ def main(): print("running pipeline" + str(pipeline_type)) convert_pipline_output_to_input(left) workflow_filename = UPSCALER_WORKFLOW - prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.UPSCALER) + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.UPSCALER, + enable_lipsync=enable_lipsync) enqueue_prompt_and_wait(prompt) output_index = output_index + 1 left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" @@ -161,7 +165,8 @@ def main(): print("running pipeline" + str(pipeline_type)) convert_pipline_output_to_input(left) workflow_filename = FACE_DETAILER_WORKFLOW - prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.FACE_DETAILER) + prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.FACE_DETAILER, + enable_lipsync=enable_lipsync) enqueue_prompt_and_wait(prompt) output_index = output_index + 1 left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4" @@ -204,7 +209,8 @@ def enqueue_prompt_and_wait(prompt): -def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, workflow_filename=None, pipeline_type = PipelineType.BASE) -> Path: +def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, workflow_filename=None, pipeline_type = PipelineType.BASE, + enable_lipsync: bool = False) -> Path: styles_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "styles" workflow_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "workflows" mappings_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "mappings" @@ -222,7 +228,9 @@ def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, work mapping_json = json.load(open(mappings_directory / mapping_json_filename)) json_mods = style_json["modifications"] - jsonpath_mods = get_jsonpath_mods(json_mods, mapping_json, pos_in=positive_prompt, neg_in=negative_prompt, pipeline_type=pipeline_type) + jsonpath_mods = get_jsonpath_mods(json_mods, mapping_json, pos_in=positive_prompt, neg_in=negative_prompt, pipeline_type=pipeline_type, + enable_lipsync=enable_lipsync) + print(*jsonpath_mods.items(), sep="\n") json_prompt = apply_jsonpath_mods(jsonpath_mods, workflow_json) @@ -232,7 +240,8 @@ def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, work def get_jsonpath_mods(style_mods: Dict[str, Any], mapping_json: Dict[str, Any], pos_in: Optional[str] = None, - neg_in: Optional[str] = None, pipeline_type = PipelineType.BASE, denoise_first_pass=1.0) -> Dict[str, Any]: + neg_in: Optional[str] = None, pipeline_type = PipelineType.BASE, denoise_first_pass=1.0, + enable_lipsync: bool = False) -> Dict[str, Any]: modifications = {} new_mod_json = style_mods.copy() # Process "loras" differently @@ -260,6 +269,9 @@ def get_jsonpath_mods(style_mods: Dict[str, Any], mapping_json: Dict[str, Any], if pipeline_type == PipelineType.IPA: new_mod_json["denoise_first_pass"] = denoise_first_pass + if enable_lipsync: + new_mod_json["cn_lips_strength"] = 1.2 + for key, value in new_mod_json.items(): mapping_key = f"$.{key}" jsonpath_expr = parse(mapping_key)