mirror of
https://github.com/storytold/vits-finetuning.git
synced 2026-10-18 07:33:53 +00:00
Initial commit (move files)
This commit is contained in:
ZDisket
+357
@@ -0,0 +1,357 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
""" Implement a pyTorch LSTM with hard sigmoid reccurent activation functions.
|
||||
Adapted from the non-cuda variant of pyTorch LSTM at
|
||||
https://github.com/pytorch/pytorch/blob/master/torch/nn/_functions/rnn.py
|
||||
"""
|
||||
|
||||
from __future__ import print_function, division
|
||||
import math
|
||||
import torch
|
||||
|
||||
from torch.nn import Module
|
||||
from torch.nn.parameter import Parameter
|
||||
from torch.nn.utils.rnn import PackedSequence
|
||||
import torch.nn.functional as F
|
||||
|
||||
class LSTMHardSigmoid(Module):
|
||||
|
||||
def __init__(self, input_size, hidden_size,
|
||||
num_layers=1, bias=True, batch_first=False,
|
||||
dropout=0, bidirectional=False):
|
||||
super(LSTMHardSigmoid, self).__init__()
|
||||
self.input_size = input_size
|
||||
self.hidden_size = hidden_size
|
||||
self.num_layers = num_layers
|
||||
self.bias = bias
|
||||
self.batch_first = batch_first
|
||||
self.dropout = dropout
|
||||
self.dropout_state = {}
|
||||
self.bidirectional = bidirectional
|
||||
num_directions = 2 if bidirectional else 1
|
||||
|
||||
gate_size = 4 * hidden_size
|
||||
|
||||
self._all_weights = []
|
||||
for layer in range(num_layers):
|
||||
for direction in range(num_directions):
|
||||
layer_input_size = input_size if layer == 0 else hidden_size * num_directions
|
||||
|
||||
w_ih = Parameter(torch.Tensor(gate_size, layer_input_size))
|
||||
w_hh = Parameter(torch.Tensor(gate_size, hidden_size))
|
||||
b_ih = Parameter(torch.Tensor(gate_size))
|
||||
b_hh = Parameter(torch.Tensor(gate_size))
|
||||
layer_params = (w_ih, w_hh, b_ih, b_hh)
|
||||
|
||||
suffix = '_reverse' if direction == 1 else ''
|
||||
param_names = ['weight_ih_l{}{}', 'weight_hh_l{}{}']
|
||||
if bias:
|
||||
param_names += ['bias_ih_l{}{}', 'bias_hh_l{}{}']
|
||||
param_names = [x.format(layer, suffix) for x in param_names]
|
||||
|
||||
for name, param in zip(param_names, layer_params):
|
||||
setattr(self, name, param)
|
||||
self._all_weights.append(param_names)
|
||||
|
||||
self.flatten_parameters()
|
||||
self.reset_parameters()
|
||||
|
||||
def flatten_parameters(self):
|
||||
"""Resets parameter data pointer so that they can use faster code paths.
|
||||
|
||||
Right now, this is a no-op wince we don't use CUDA acceleration.
|
||||
"""
|
||||
self._data_ptrs = []
|
||||
|
||||
def _apply(self, fn):
|
||||
ret = super(LSTMHardSigmoid, self)._apply(fn)
|
||||
self.flatten_parameters()
|
||||
return ret
|
||||
|
||||
def reset_parameters(self):
|
||||
stdv = 1.0 / math.sqrt(self.hidden_size)
|
||||
for weight in self.parameters():
|
||||
weight.data.uniform_(-stdv, stdv)
|
||||
|
||||
def forward(self, input, hx=None):
|
||||
is_packed = isinstance(input, PackedSequence)
|
||||
if is_packed:
|
||||
batch_sizes = input.batch_sizes
|
||||
input = input.data
|
||||
max_batch_size = batch_sizes[0]
|
||||
else:
|
||||
batch_sizes = None
|
||||
max_batch_size = input.size(0) if self.batch_first else input.size(1)
|
||||
|
||||
if hx is None:
|
||||
num_directions = 2 if self.bidirectional else 1
|
||||
hx = torch.autograd.Variable(input.data.new(self.num_layers *
|
||||
num_directions,
|
||||
max_batch_size,
|
||||
self.hidden_size).zero_(), requires_grad=False)
|
||||
hx = (hx, hx)
|
||||
|
||||
has_flat_weights = list(p.data.data_ptr() for p in self.parameters()) == self._data_ptrs
|
||||
if has_flat_weights:
|
||||
first_data = next(self.parameters()).data
|
||||
assert first_data.storage().size() == self._param_buf_size
|
||||
flat_weight = first_data.new().set_(first_data.storage(), 0, torch.Size([self._param_buf_size]))
|
||||
else:
|
||||
flat_weight = None
|
||||
func = AutogradRNN(
|
||||
self.input_size,
|
||||
self.hidden_size,
|
||||
num_layers=self.num_layers,
|
||||
batch_first=self.batch_first,
|
||||
dropout=self.dropout,
|
||||
train=self.training,
|
||||
bidirectional=self.bidirectional,
|
||||
batch_sizes=batch_sizes,
|
||||
dropout_state=self.dropout_state,
|
||||
flat_weight=flat_weight
|
||||
)
|
||||
output, hidden = func(input, self.all_weights, hx)
|
||||
if is_packed:
|
||||
output = PackedSequence(output, batch_sizes)
|
||||
return output, hidden
|
||||
|
||||
def __repr__(self):
|
||||
s = '{name}({input_size}, {hidden_size}'
|
||||
if self.num_layers != 1:
|
||||
s += ', num_layers={num_layers}'
|
||||
if self.bias is not True:
|
||||
s += ', bias={bias}'
|
||||
if self.batch_first is not False:
|
||||
s += ', batch_first={batch_first}'
|
||||
if self.dropout != 0:
|
||||
s += ', dropout={dropout}'
|
||||
if self.bidirectional is not False:
|
||||
s += ', bidirectional={bidirectional}'
|
||||
s += ')'
|
||||
return s.format(name=self.__class__.__name__, **self.__dict__)
|
||||
|
||||
def __setstate__(self, d):
|
||||
super(LSTMHardSigmoid, self).__setstate__(d)
|
||||
self.__dict__.setdefault('_data_ptrs', [])
|
||||
if 'all_weights' in d:
|
||||
self._all_weights = d['all_weights']
|
||||
if isinstance(self._all_weights[0][0], str):
|
||||
return
|
||||
num_layers = self.num_layers
|
||||
num_directions = 2 if self.bidirectional else 1
|
||||
self._all_weights = []
|
||||
for layer in range(num_layers):
|
||||
for direction in range(num_directions):
|
||||
suffix = '_reverse' if direction == 1 else ''
|
||||
weights = ['weight_ih_l{}{}', 'weight_hh_l{}{}', 'bias_ih_l{}{}', 'bias_hh_l{}{}']
|
||||
weights = [x.format(layer, suffix) for x in weights]
|
||||
if self.bias:
|
||||
self._all_weights += [weights]
|
||||
else:
|
||||
self._all_weights += [weights[:2]]
|
||||
|
||||
@property
|
||||
def all_weights(self):
|
||||
return [[getattr(self, weight) for weight in weights] for weights in self._all_weights]
|
||||
|
||||
def AutogradRNN(input_size, hidden_size, num_layers=1, batch_first=False,
|
||||
dropout=0, train=True, bidirectional=False, batch_sizes=None,
|
||||
dropout_state=None, flat_weight=None):
|
||||
|
||||
cell = LSTMCell
|
||||
|
||||
if batch_sizes is None:
|
||||
rec_factory = Recurrent
|
||||
else:
|
||||
rec_factory = variable_recurrent_factory(batch_sizes)
|
||||
|
||||
if bidirectional:
|
||||
layer = (rec_factory(cell), rec_factory(cell, reverse=True))
|
||||
else:
|
||||
layer = (rec_factory(cell),)
|
||||
|
||||
func = StackedRNN(layer,
|
||||
num_layers,
|
||||
True,
|
||||
dropout=dropout,
|
||||
train=train)
|
||||
|
||||
def forward(input, weight, hidden):
|
||||
if batch_first and batch_sizes is None:
|
||||
input = input.transpose(0, 1)
|
||||
|
||||
nexth, output = func(input, hidden, weight)
|
||||
|
||||
if batch_first and batch_sizes is None:
|
||||
output = output.transpose(0, 1)
|
||||
|
||||
return output, nexth
|
||||
|
||||
return forward
|
||||
|
||||
def Recurrent(inner, reverse=False):
|
||||
def forward(input, hidden, weight):
|
||||
output = []
|
||||
steps = range(input.size(0) - 1, -1, -1) if reverse else range(input.size(0))
|
||||
for i in steps:
|
||||
hidden = inner(input[i], hidden, *weight)
|
||||
# hack to handle LSTM
|
||||
output.append(hidden[0] if isinstance(hidden, tuple) else hidden)
|
||||
|
||||
if reverse:
|
||||
output.reverse()
|
||||
output = torch.cat(output, 0).view(input.size(0), *output[0].size())
|
||||
|
||||
return hidden, output
|
||||
|
||||
return forward
|
||||
|
||||
|
||||
def variable_recurrent_factory(batch_sizes):
|
||||
def fac(inner, reverse=False):
|
||||
if reverse:
|
||||
return VariableRecurrentReverse(batch_sizes, inner)
|
||||
else:
|
||||
return VariableRecurrent(batch_sizes, inner)
|
||||
return fac
|
||||
|
||||
def VariableRecurrent(batch_sizes, inner):
|
||||
def forward(input, hidden, weight):
|
||||
output = []
|
||||
input_offset = 0
|
||||
last_batch_size = batch_sizes[0]
|
||||
hiddens = []
|
||||
flat_hidden = not isinstance(hidden, tuple)
|
||||
if flat_hidden:
|
||||
hidden = (hidden,)
|
||||
for batch_size in batch_sizes:
|
||||
step_input = input[input_offset:input_offset + batch_size]
|
||||
input_offset += batch_size
|
||||
|
||||
dec = last_batch_size - batch_size
|
||||
if dec > 0:
|
||||
hiddens.append(tuple(h[-dec:] for h in hidden))
|
||||
hidden = tuple(h[:-dec] for h in hidden)
|
||||
last_batch_size = batch_size
|
||||
|
||||
if flat_hidden:
|
||||
hidden = (inner(step_input, hidden[0], *weight),)
|
||||
else:
|
||||
hidden = inner(step_input, hidden, *weight)
|
||||
|
||||
output.append(hidden[0])
|
||||
hiddens.append(hidden)
|
||||
hiddens.reverse()
|
||||
|
||||
hidden = tuple(torch.cat(h, 0) for h in zip(*hiddens))
|
||||
assert hidden[0].size(0) == batch_sizes[0]
|
||||
if flat_hidden:
|
||||
hidden = hidden[0]
|
||||
output = torch.cat(output, 0)
|
||||
|
||||
return hidden, output
|
||||
|
||||
return forward
|
||||
|
||||
|
||||
def VariableRecurrentReverse(batch_sizes, inner):
|
||||
def forward(input, hidden, weight):
|
||||
output = []
|
||||
input_offset = input.size(0)
|
||||
last_batch_size = batch_sizes[-1]
|
||||
initial_hidden = hidden
|
||||
flat_hidden = not isinstance(hidden, tuple)
|
||||
if flat_hidden:
|
||||
hidden = (hidden,)
|
||||
initial_hidden = (initial_hidden,)
|
||||
hidden = tuple(h[:batch_sizes[-1]] for h in hidden)
|
||||
for batch_size in reversed(batch_sizes):
|
||||
inc = batch_size - last_batch_size
|
||||
if inc > 0:
|
||||
hidden = tuple(torch.cat((h, ih[last_batch_size:batch_size]), 0)
|
||||
for h, ih in zip(hidden, initial_hidden))
|
||||
last_batch_size = batch_size
|
||||
step_input = input[input_offset - batch_size:input_offset]
|
||||
input_offset -= batch_size
|
||||
|
||||
if flat_hidden:
|
||||
hidden = (inner(step_input, hidden[0], *weight),)
|
||||
else:
|
||||
hidden = inner(step_input, hidden, *weight)
|
||||
output.append(hidden[0])
|
||||
|
||||
output.reverse()
|
||||
output = torch.cat(output, 0)
|
||||
if flat_hidden:
|
||||
hidden = hidden[0]
|
||||
return hidden, output
|
||||
|
||||
return forward
|
||||
|
||||
def StackedRNN(inners, num_layers, lstm=False, dropout=0, train=True):
|
||||
|
||||
num_directions = len(inners)
|
||||
total_layers = num_layers * num_directions
|
||||
|
||||
def forward(input, hidden, weight):
|
||||
assert(len(weight) == total_layers)
|
||||
next_hidden = []
|
||||
|
||||
if lstm:
|
||||
hidden = list(zip(*hidden))
|
||||
|
||||
for i in range(num_layers):
|
||||
all_output = []
|
||||
for j, inner in enumerate(inners):
|
||||
l = i * num_directions + j
|
||||
|
||||
hy, output = inner(input, hidden[l], weight[l])
|
||||
next_hidden.append(hy)
|
||||
all_output.append(output)
|
||||
|
||||
input = torch.cat(all_output, input.dim() - 1)
|
||||
|
||||
if dropout != 0 and i < num_layers - 1:
|
||||
input = F.dropout(input, p=dropout, training=train, inplace=False)
|
||||
|
||||
if lstm:
|
||||
next_h, next_c = zip(*next_hidden)
|
||||
next_hidden = (
|
||||
torch.cat(next_h, 0).view(total_layers, *next_h[0].size()),
|
||||
torch.cat(next_c, 0).view(total_layers, *next_c[0].size())
|
||||
)
|
||||
else:
|
||||
next_hidden = torch.cat(next_hidden, 0).view(
|
||||
total_layers, *next_hidden[0].size())
|
||||
|
||||
return next_hidden, input
|
||||
|
||||
return forward
|
||||
|
||||
def LSTMCell(input, hidden, w_ih, w_hh, b_ih=None, b_hh=None):
|
||||
"""
|
||||
A modified LSTM cell with hard sigmoid activation on the input, forget and output gates.
|
||||
"""
|
||||
hx, cx = hidden
|
||||
gates = F.linear(input, w_ih, b_ih) + F.linear(hx, w_hh, b_hh)
|
||||
|
||||
ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)
|
||||
|
||||
ingate = hard_sigmoid(ingate)
|
||||
forgetgate = hard_sigmoid(forgetgate)
|
||||
cellgate = torch.tanh(cellgate)
|
||||
outgate = hard_sigmoid(outgate)
|
||||
|
||||
cy = (forgetgate * cx) + (ingate * cellgate)
|
||||
hy = outgate * torch.tanh(cy)
|
||||
|
||||
return hy, cy
|
||||
|
||||
def hard_sigmoid(x):
|
||||
"""
|
||||
Computes element-wise hard sigmoid of x.
|
||||
See e.g. https://github.com/Theano/Theano/blob/master/theano/tensor/nnet/sigm.py#L279
|
||||
"""
|
||||
x = (0.2 * x) + 0.5
|
||||
x = F.threshold(-x, -1, -1)
|
||||
x = F.threshold(-x, 0, 0)
|
||||
return x
|
||||
Reference in New Issue
Block a user