mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-17 00:23:14 +00:00
Brandon Thomas
046ab789c0
https://lucaria-academy.github.io/Animate-X/ https://github.com/antgroup/animate-x @ fdc80909f911d8e487cb4e6847f2e4c6501b62af
304 lines
10 KiB
Python
304 lines
10 KiB
Python
import os
|
|
import torch
|
|
import logging
|
|
import open_clip
|
|
import numpy as np
|
|
import torch.nn as nn
|
|
import torchvision.transforms as T
|
|
|
|
from utils.registry_class import EMBEDDER
|
|
import kornia
|
|
|
|
|
|
@EMBEDDER.register_class()
|
|
class FrozenOpenCLIPEmbedder(nn.Module):
|
|
"""
|
|
Uses the OpenCLIP transformer encoder for text
|
|
"""
|
|
LAYERS = [
|
|
#"pooled",
|
|
"last",
|
|
"penultimate"
|
|
]
|
|
def __init__(self, pretrained, arch="ViT-H-14", device="cuda", max_length=77,
|
|
freeze=True, layer="last"):
|
|
super().__init__()
|
|
assert layer in self.LAYERS
|
|
model, _, _ = open_clip.create_model_and_transforms(arch, device=torch.device('cpu'), pretrained=pretrained)
|
|
del model.visual
|
|
self.model = model
|
|
|
|
self.device = device
|
|
self.max_length = max_length
|
|
if freeze:
|
|
self.freeze()
|
|
self.layer = layer
|
|
if self.layer == "last":
|
|
self.layer_idx = 0
|
|
elif self.layer == "penultimate":
|
|
self.layer_idx = 1
|
|
else:
|
|
raise NotImplementedError()
|
|
|
|
def freeze(self):
|
|
self.model = self.model.eval()
|
|
for param in self.parameters():
|
|
param.requires_grad = False
|
|
|
|
def forward(self, text):
|
|
tokens = open_clip.tokenize(text)
|
|
z = self.encode_with_transformer(tokens.to(self.device))
|
|
return z
|
|
|
|
def encode_with_transformer(self, text):
|
|
x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model]
|
|
x = x + self.model.positional_embedding
|
|
x = x.permute(1, 0, 2) # NLD -> LND
|
|
x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask)
|
|
x = x.permute(1, 0, 2) # LND -> NLD
|
|
x = self.model.ln_final(x)
|
|
return x
|
|
|
|
def text_transformer_forward(self, x: torch.Tensor, attn_mask = None):
|
|
for i, r in enumerate(self.model.transformer.resblocks):
|
|
if i == len(self.model.transformer.resblocks) - self.layer_idx:
|
|
break
|
|
if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting():
|
|
x = checkpoint(r, x, attn_mask)
|
|
else:
|
|
x = r(x, attn_mask=attn_mask)
|
|
return x
|
|
|
|
def encode(self, text):
|
|
return self(text)
|
|
|
|
|
|
@EMBEDDER.register_class()
|
|
class FrozenOpenCLIPVisualEmbedder(nn.Module):
|
|
"""
|
|
Uses the OpenCLIP transformer encoder for text
|
|
"""
|
|
LAYERS = [
|
|
#"pooled",
|
|
"last",
|
|
"penultimate"
|
|
]
|
|
def __init__(self, pretrained, vit_resolution=(224, 224), arch="ViT-H-14", device="cuda", max_length=77,
|
|
freeze=True, layer="last"):
|
|
super().__init__()
|
|
assert layer in self.LAYERS
|
|
model, _, preprocess = open_clip.create_model_and_transforms(
|
|
arch, device=torch.device('cpu'), pretrained=pretrained)
|
|
|
|
del model.transformer
|
|
self.model = model
|
|
data_white = np.ones((vit_resolution[0], vit_resolution[1], 3), dtype=np.uint8)*255
|
|
self.white_image = preprocess(T.ToPILImage()(data_white)).unsqueeze(0)
|
|
|
|
self.device = device
|
|
self.max_length = max_length # 77
|
|
if freeze:
|
|
self.freeze()
|
|
self.layer = layer # 'penultimate'
|
|
if self.layer == "last":
|
|
self.layer_idx = 0
|
|
elif self.layer == "penultimate":
|
|
self.layer_idx = 1
|
|
else:
|
|
raise NotImplementedError()
|
|
|
|
def freeze(self):
|
|
self.model = self.model.eval()
|
|
for param in self.parameters():
|
|
param.requires_grad = False
|
|
|
|
def forward(self, image):
|
|
# tokens = open_clip.tokenize(text)
|
|
z = self.model.encode_image(image.to(self.device))
|
|
return z
|
|
|
|
def encode_with_transformer(self, text):
|
|
x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model]
|
|
x = x + self.model.positional_embedding
|
|
x = x.permute(1, 0, 2) # NLD -> LND
|
|
x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask)
|
|
x = x.permute(1, 0, 2) # LND -> NLD
|
|
x = self.model.ln_final(x)
|
|
|
|
return x
|
|
|
|
def text_transformer_forward(self, x: torch.Tensor, attn_mask = None):
|
|
for i, r in enumerate(self.model.transformer.resblocks):
|
|
if i == len(self.model.transformer.resblocks) - self.layer_idx:
|
|
break
|
|
if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting():
|
|
x = checkpoint(r, x, attn_mask)
|
|
else:
|
|
x = r(x, attn_mask=attn_mask)
|
|
return x
|
|
|
|
def encode(self, text):
|
|
return self(text)
|
|
|
|
|
|
|
|
@EMBEDDER.register_class()
|
|
class FrozenOpenCLIPTextVisualEmbedder(nn.Module):
|
|
"""
|
|
Uses the OpenCLIP transformer encoder for text
|
|
"""
|
|
LAYERS = [
|
|
#"pooled",
|
|
"last",
|
|
"penultimate"
|
|
]
|
|
def __init__(self, pretrained, arch="ViT-H-14", device="cuda", max_length=77,
|
|
freeze=True, layer="last", **kwargs):
|
|
super().__init__()
|
|
assert layer in self.LAYERS
|
|
model, _, _ = open_clip.create_model_and_transforms(arch, device=torch.device('cpu'), pretrained=pretrained)
|
|
self.model = model
|
|
|
|
self.device = device
|
|
self.max_length = max_length
|
|
if freeze:
|
|
self.freeze()
|
|
self.layer = layer
|
|
if self.layer == "last":
|
|
self.layer_idx = 0
|
|
elif self.layer == "penultimate":
|
|
self.layer_idx = 1
|
|
else:
|
|
raise NotImplementedError()
|
|
|
|
def freeze(self):
|
|
self.model = self.model.eval()
|
|
for param in self.parameters():
|
|
param.requires_grad = False
|
|
|
|
|
|
def forward(self, image=None, text=None):
|
|
|
|
xi = self.model.encode_image(image.to(self.device)) if image is not None else None
|
|
tokens = open_clip.tokenize(text)
|
|
xt, x = self.encode_with_transformer(tokens.to(self.device))
|
|
return xi, xt, x
|
|
|
|
def encode_with_transformer(self, text):
|
|
x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model]
|
|
x = x + self.model.positional_embedding
|
|
x = x.permute(1, 0, 2) # NLD -> LND
|
|
x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask)
|
|
x = x.permute(1, 0, 2) # LND -> NLD
|
|
x = self.model.ln_final(x)
|
|
xt = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.model.text_projection
|
|
return xt, x
|
|
|
|
|
|
def encode_image(self, image):
|
|
return self.model.visual(image)
|
|
|
|
def text_transformer_forward(self, x: torch.Tensor, attn_mask = None):
|
|
for i, r in enumerate(self.model.transformer.resblocks):
|
|
if i == len(self.model.transformer.resblocks) - self.layer_idx:
|
|
break
|
|
if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting():
|
|
x = checkpoint(r, x, attn_mask)
|
|
else:
|
|
x = r(x, attn_mask=attn_mask)
|
|
return x
|
|
|
|
def encode(self, text):
|
|
|
|
return self(text)
|
|
|
|
|
|
|
|
|
|
class AbstractEncoder(nn.Module):
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
def encode(self, *args, **kwargs):
|
|
raise NotImplementedError
|
|
|
|
class FrozenOpenCLIPImageEmbedderV2(AbstractEncoder):
|
|
"""
|
|
Uses the OpenCLIP vision transformer encoder for images
|
|
"""
|
|
|
|
def __init__(self, pretrained="", arch="ViT-H-14", device="cuda",
|
|
freeze=True, layer="pooled", antialias=True):
|
|
super().__init__()
|
|
model, _, _ = open_clip.create_model_and_transforms(arch, device=torch.device('cpu'),
|
|
pretrained=pretrained, )
|
|
del model.transformer
|
|
self.model = model
|
|
self.device = device
|
|
|
|
if freeze:
|
|
self.freeze()
|
|
self.layer = layer
|
|
if self.layer == "penultimate":
|
|
raise NotImplementedError()
|
|
self.layer_idx = 1
|
|
|
|
self.antialias = antialias
|
|
|
|
self.register_buffer('mean', torch.Tensor([0.48145466, 0.4578275, 0.40821073]), persistent=False)
|
|
self.register_buffer('std', torch.Tensor([0.26862954, 0.26130258, 0.27577711]), persistent=False)
|
|
|
|
|
|
def preprocess(self, x):
|
|
# normalize to [0,1]
|
|
x = kornia.geometry.resize(x, (224, 224),
|
|
interpolation='bicubic', align_corners=True,
|
|
antialias=self.antialias)
|
|
x = (x + 1.) / 2.
|
|
# renormalize according to clip
|
|
x = kornia.enhance.normalize(x, self.mean, self.std)
|
|
return x
|
|
|
|
def freeze(self):
|
|
self.model = self.model.eval()
|
|
for param in self.model.parameters():
|
|
param.requires_grad = False
|
|
|
|
def forward(self, image, no_dropout=False):
|
|
## image: b c h w
|
|
z = self.encode_with_vision_transformer(image)
|
|
return z
|
|
|
|
def encode_with_vision_transformer(self, x):
|
|
x = self.preprocess(x)
|
|
|
|
# to patches - whether to use dual patchnorm
|
|
if self.model.visual.input_patchnorm:
|
|
# einops - rearrange(x, 'b c (h p1) (w p2) -> b (h w) (c p1 p2)')
|
|
x = x.reshape(x.shape[0], x.shape[1], self.model.visual.grid_size[0], self.model.visual.patch_size[0], self.model.visual.grid_size[1], self.model.visual.patch_size[1])
|
|
x = x.permute(0, 2, 4, 1, 3, 5)
|
|
x = x.reshape(x.shape[0], self.model.visual.grid_size[0] * self.model.visual.grid_size[1], -1)
|
|
x = self.model.visual.patchnorm_pre_ln(x)
|
|
x = self.model.visual.conv1(x)
|
|
else:
|
|
x = self.model.visual.conv1(x) # shape = [*, width, grid, grid]
|
|
x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2]
|
|
x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
|
|
|
|
# class embeddings and positional embeddings
|
|
x = torch.cat(
|
|
[self.model.visual.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
|
|
x], dim=1) # shape = [*, grid ** 2 + 1, width]
|
|
x = x + self.model.visual.positional_embedding.to(x.dtype)
|
|
|
|
# a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in
|
|
x = self.model.visual.patch_dropout(x)
|
|
x = self.model.visual.ln_pre(x)
|
|
|
|
x = x.permute(1, 0, 2) # NLD -> LND
|
|
x = self.model.visual.transformer(x)
|
|
x = x.permute(1, 0, 2) # LND -> NLD
|
|
|
|
return x
|
|
|