FineTrainers-Conditioning/conditioning/patchify_technique.py

import torch

# single channel latent 4608x128 patchified
# doubled channel ... 
# torch.Size([1, 4608, 256]) 
# modify the linear layer ... (4608x256 and 128x2048) 

# Pipeline
# Video Input
# Video tensor torch.Size([1, 96, 3, 512, 768]) B F C H W
# to 
# Video tensor permuted torch.Size([1, 3, 96, 512, 768]) B C F H W

# Latent Encode temporal frames 8 compression 32 compression spatio
# channels go to 128
# torch.Size([1, 128, 12, 16, 24])

# Creating a Multi Channel Single Video
a = torch.randn(1,2,2,2) # video
print(a)

a = a.flatten(0,3)
print(a)
print(a.shape)

# b = torch.randn(1,2,4,4) # conditioning
# c = torch.cat([a,b],dim=1)

# Final shape.
# print(c.shape)
# print(c)


#_prepare_latents
# prepare_latents


# ImageOrVideoDataset Shape result 
# _preprocess_video [F, C, H, W] video tensor 4 

# _preprocess_image [1, C, H, W] (1-frame video)

# VAE Latent Shape Results Dims
# 4 second video 24 fps 
# torch.Size([1, 128, 12, 16, 24])


# VideoProcessor latent output shape preprocess_video

# Video Tensor Shape 

# Latent Tensor Shape 

# VAE Patchification Result Shape

# Unpacked latents of shape are [B, C, F, H, W] are patched into tokens of shape [B, C, F // p_t, p_t, H // p, p, W // p, p].
# The patch dimensions are then permuted and collapsed into the channel dimension of shape:
# [B, F // p_t * H // p * W // p, C * p_t * p * p] (an ndim=3 tensor).
# dim=0 is the batch size, dim=1 is the effective video sequence length, dim=2 is the effective number of input features

# encode and then decode and decouple the videos

# check transformer hidden state, should still be same sequence or slightly larger sequence.

def patchify_multichannel_latent(video_latent:torch.tensor,patch_size:int):# B, C, F, W, H 
    pass 

def unpatchify_multichannel_laten(video_latent:torch.tensor,patch_size:int): # B, C ,F, W, H
    pass

def mask_multichannel_latent(mask_shape:torch.tensor):
    pass