Adding Conditioning Blocks to the Diffusion Transformer.

conditioning/patchify_technique.py

@@ -0,0 +1,72 @@
+import torch
+
+# single channel latent 4608x128 patchified
+# doubled channel ... 
+# torch.Size([1, 4608, 256]) 
+# modify the linear layer ... (4608x256 and 128x2048) 
+
+# Pipeline
+# Video Input
+# Video tensor torch.Size([1, 96, 3, 512, 768]) B F C H W
+# to 
+# Video tensor permuted torch.Size([1, 3, 96, 512, 768]) B C F H W
+
+# Latent Encode temporal frames 8 compression 32 compression spatio
+# channels go to 128
+# torch.Size([1, 128, 12, 16, 24])
+
+# Creating a Multi Channel Single Video
+a = torch.randn(1,2,2,2) # video
+print(a)
+
+a = a.flatten(0,3)
+print(a)
+print(a.shape)
+
+# b = torch.randn(1,2,4,4) # conditioning
+# c = torch.cat([a,b],dim=1)
+
+# Final shape.
+# print(c.shape)
+# print(c)
+
+
+#_prepare_latents
+# prepare_latents
+
+
+# ImageOrVideoDataset Shape result 
+# _preprocess_video [F, C, H, W] video tensor 4 
+
+# _preprocess_image [1, C, H, W] (1-frame video)
+
+# VAE Latent Shape Results Dims
+# 4 second video 24 fps 
+# torch.Size([1, 128, 12, 16, 24])
+
+
+# VideoProcessor latent output shape preprocess_video
+
+# Video Tensor Shape 
+
+# Latent Tensor Shape 
+
+# VAE Patchification Result Shape
+
+# Unpacked latents of shape are [B, C, F, H, W] are patched into tokens of shape [B, C, F // p_t, p_t, H // p, p, W // p, p].
+# The patch dimensions are then permuted and collapsed into the channel dimension of shape:
+# [B, F // p_t * H // p * W // p, C * p_t * p * p] (an ndim=3 tensor).
+# dim=0 is the batch size, dim=1 is the effective video sequence length, dim=2 is the effective number of input features
+
+# encode and then decode and decouple the videos
+
+# check transformer hidden state, should still be same sequence or slightly larger sequence.
+
+def patchify_multichannel_latent(video_latent:torch.tensor,patch_size:int):# B, C, F, W, H 
+    pass 
+
+def unpatchify_multichannel_laten(video_latent:torch.tensor,patch_size:int): # B, C ,F, W, H
+    pass
+
+def mask_multichannel_latent(mask_shape:torch.tensor):
+    pass