storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-17 00:23:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

arg to enable lipsync

Browse Source
This commit is contained in:
Brandon Thomas
2024-05-21 21:22:31 -04:00
parent ee352ceca2
commit 4c6c05a781
1 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workflows/comfy/ComfyLauncher/ComfyRunnerForSidecar.py
+18 -6
View File
@@ -92,6 +92,7 @@ def parse_args():
parser.add_argument('--pause-between-steps', type=int, help='pause between steps', required=False, default=1)
parser.add_argument('--face-detailer-enabled', help='face detailer enabled', required=False, default=False, action=argparse.BooleanOptionalAction)
parser.add_argument('--upscaler-enabled', help='upscale enabled', required=False, default=False, action=argparse.BooleanOptionalAction)
parser.add_argument('--lipsync-enabled', help='lipsync enabled', required=False, default=False, action=argparse.BooleanOptionalAction)
parser.add_argument('--strength', type=float, help='strength', required=False, default=1.0)
return parser.parse_args()
@@ -133,6 +134,7 @@ def main():
positive_prompt = None
negative_prompt = None
workflow_filename = None
enable_lipsync = args.lipsync_enabled
if pipeline_type == PipelineType.IPA:
workflow_filename = MAIN_IPA_WORKFLOW
if positive_prompt_filename:
@@ -142,7 +144,8 @@ def main():
validate_style_name(style)
print("running pipeline" + str(pipeline_type))
# ensure_pipeline_input_present()
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename)
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename,
enable_lipsync=enable_lipsync)
enqueue_prompt_and_wait(prompt)
output_index = output_index + 1
left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4"
@@ -151,7 +154,8 @@ def main():
print("running pipeline" + str(pipeline_type))
convert_pipline_output_to_input(left)
workflow_filename = UPSCALER_WORKFLOW
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.UPSCALER)
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.UPSCALER,
enable_lipsync=enable_lipsync)
enqueue_prompt_and_wait(prompt)
output_index = output_index + 1
left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4"
@@ -161,7 +165,8 @@ def main():
print("running pipeline" + str(pipeline_type))
convert_pipline_output_to_input(left)
workflow_filename = FACE_DETAILER_WORKFLOW
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.FACE_DETAILER)
prompt = generate_prompt_for_style(style, positive_prompt, negative_prompt, workflow_filename, PipelineType.FACE_DETAILER,
enable_lipsync=enable_lipsync)
enqueue_prompt_and_wait(prompt)
output_index = output_index + 1
left = Path(COMFY_ROOT_DIRECTORY) / "output" / f"vid2vid/SparseUpscaleInterp_0000{output_index}.mp4"
@@ -204,7 +209,8 @@ def enqueue_prompt_and_wait(prompt):
def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, workflow_filename=None, pipeline_type = PipelineType.BASE) -> Path:
def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, workflow_filename=None, pipeline_type = PipelineType.BASE,
enable_lipsync: bool = False) -> Path:
styles_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "styles"
workflow_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "workflows"
mappings_directory = Path(V2V_WORKFLOWS_DIRECTORY) / "mappings"
@@ -222,7 +228,9 @@ def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, work
mapping_json = json.load(open(mappings_directory / mapping_json_filename))
json_mods = style_json["modifications"]
jsonpath_mods = get_jsonpath_mods(json_mods, mapping_json, pos_in=positive_prompt, neg_in=negative_prompt, pipeline_type=pipeline_type)
jsonpath_mods = get_jsonpath_mods(json_mods, mapping_json, pos_in=positive_prompt, neg_in=negative_prompt, pipeline_type=pipeline_type,
enable_lipsync=enable_lipsync)
print(*jsonpath_mods.items(), sep="\n")
json_prompt = apply_jsonpath_mods(jsonpath_mods, workflow_json)
@@ -232,7 +240,8 @@ def generate_prompt_for_style(style_name, positive_prompt, negative_prompt, work
def get_jsonpath_mods(style_mods: Dict[str, Any], mapping_json: Dict[str, Any], pos_in: Optional[str] = None,
neg_in: Optional[str] = None, pipeline_type = PipelineType.BASE, denoise_first_pass=1.0) -> Dict[str, Any]:
neg_in: Optional[str] = None, pipeline_type = PipelineType.BASE, denoise_first_pass=1.0,
enable_lipsync: bool = False) -> Dict[str, Any]:
modifications = {}
new_mod_json = style_mods.copy()
# Process "loras" differently
@@ -260,6 +269,9 @@ def get_jsonpath_mods(style_mods: Dict[str, Any], mapping_json: Dict[str, Any],
if pipeline_type == PipelineType.IPA:
new_mod_json["denoise_first_pass"] = denoise_first_pass
if enable_lipsync:
new_mod_json["cn_lips_strength"] = 1.2
for key, value in new_mod_json.items():
mapping_key = f"$.{key}"
jsonpath_expr = parse(mapping_key)