Checking inputs.

finetrainers/conditioning/input_test.py

@@ -0,0 +1,83 @@
+import torch
+from diffusers import LTXPipeline
+from diffusers import LTXVideoTransformer3DModel
+from diffusers.utils import export_to_video
+from diffusers.utils import testing_utils
+from finetrainers.dataset import ImageOrVideoDataset
+from finetrainers.dataset import BucketSampler, ImageOrVideoDatasetWithResizing, PrecomputedDataset
+from finetrainers.ltx_video.full_finetune_condition import collate_fn_t2v_cond
+from finetrainers.conditioning import condition_latents_prepare,post_conditioned_latent_patchify
+from diffusers.video_processor import VideoProcessor
+from diffusers.pipelines.ltx import LTXPipeine 
+
+testing_utils.enable_full_determinism()
+device = "cuda"
+
+pipe = LTXPipeline.from_pretrained(
+    "Lightricks/LTX-Video", torch_dtype=torch.bfloat16,
+    # transformer=transformer
+).to(device)
+dtype = torch.bfloat16
+
+vae_spatial_compression_ratio = 32
+vae_temporal_compression_ratio = 8
+transformer_temporal_patch_size = 1
+transformer_spatial_patch_size = 1
+
+video_processor = VideoProcessor(vae_scale_factor=vae_spatial_compression_ratio)
+
+generator=torch.Generator(device=device).manual_seed(42),
+
+vae = pipe.vae
+
+dataset = ImageOrVideoDataset(
+    data_root="${workspaceFolder}/video-dataset-pose-test",
+    caption_column="prompts.txt",
+    video_column="videos.txt", 
+    resolution_buckets=[(96,512,768), (168,512,768)],
+    id_token="",
+    pose_column="poses.txt"
+)
+
+dataloader = torch.utils.data.DataLoader(
+    dataset,
+    batch_size=1,
+    sampler=BucketSampler(dataset, batch_size=1, shuffle=True),
+    collate_fn=collate_fn_t2v_cond,
+    num_workers=1,
+    pin_memory=True,
+)
+
+for step, batch in enumerate(dataloader):
+    videos = batch["videos"]
+    prompts = batch["prompts"]
+    poses = batch["poses"]
+    # generate video across all frames
+    img_refs = batch["img_refs"]
+
+
+# single img video ref ...
+video_tensor = dataset._preprocess_video_image_reference_video()
+
+video_latent = condition_latents_prepare.prepare_latents_for_conditioning(image_or_video=video_tensor)
+
+tokens = condition_latents_prepare.post_conditioned_latent_patchify()
+
+
+latents = LTXPipeine._unpack_latents(
+                tokens,
+                latent_num_frames,
+                latent_height,
+                latent_width,
+                transformer_spatial_patch_size,
+                transformer_temporal_patch_size,
+            )
+
+latents = LTXPipeine._denormalize_latents(
+    latents, vae.latents_mean, vae.latents_std, vae.config.scaling_factor
+)
+
+
+
+video = vae.decode(latents, None, return_dict=False)[0]
+video_processor.postprocess_video(video,output_type="pil")

finetrainers/models.py

@@ -12,7 +12,7 @@ SUPPORTED_MODEL_CONFIGS = {
     },
     "ltx_video": {
         "lora": LTX_VIDEO_T2V_LORA_CONFIG,
-        "full-finetune":LTX_VIDEO_COND_T2V_FULL_FINETUNE_CONFIG # LTX_VIDEO_T2V_FULL_FINETUNE_CONFIG,
+        "full-finetune": LTX_VIDEO_COND_T2V_FULL_FINETUNE_CONFIG, #
     },
     "cogvideox": {
         "lora": COGVIDEOX_T2V_LORA_CONFIG,