mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-25 08:40:31 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,123 @@
|
||||
name: &name "Wav2vec_Pretrain"
|
||||
# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py
|
||||
# These parameters are based off the FairSeq implementation.
|
||||
# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml
|
||||
model:
|
||||
transformer_encoder:
|
||||
dropout: 0.1 # Dropout probability for the transformer
|
||||
conv: # Config for convolutional model that generates positional embeddings required for attention layer
|
||||
conv_pos: 128 # Number of filters for convolutional positional embeddings
|
||||
conv_pos_groups: 16 # Number of groups for convolutional positional embeddings
|
||||
encoder:
|
||||
encoder_layers: 12 # Number of encoder layers in transformer model
|
||||
encoder_layerdrop: 0.05 # Probability of dropping transformer layers
|
||||
embedding_dim: 768 # Encoder embedding dim
|
||||
ffn_embedding_dim: 3072 # Encoder embedding dim for feed forward
|
||||
num_attention_heads: 8 # Number of encoder attention heads
|
||||
dropout: 0.1 # Dropout probability for transformer encoder
|
||||
activation_fn: gelu # Activation for transformer
|
||||
layer_norm_first: False # Apply layer norm first within the transformer
|
||||
loss:
|
||||
prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss
|
||||
feature_loss_weight: 0 # Weight applied to feature L2 Norm
|
||||
quantizer:
|
||||
quantize_targets: True # Quantize targets
|
||||
quantize_input: False # Quantize inputs
|
||||
same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled)
|
||||
latent_vars: 320 # Number of latent variables in each group of the codebook
|
||||
latent_groups: 2 # Number of groups within the codebook
|
||||
latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups
|
||||
latent_temp: [2, 0.5, 0.999995] # Quantize temperature (start, stop, decay factor)
|
||||
conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features
|
||||
extractor_mode: default # Mode for feature extractor. [Default, layer_norm]
|
||||
conv_bias: False # Include bias in convolution feature extractor model
|
||||
masking:
|
||||
mask_prob: 0.65 # Probability of replacing token with mask
|
||||
mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson]
|
||||
mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices)
|
||||
mask_length: 10 # Length of mask when masking time steps
|
||||
no_mask_overlap: False # Whether to allow masks to overlap
|
||||
mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled)
|
||||
mask_channel_prob: 0 # Probability of replacing a feature with 0
|
||||
mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson]
|
||||
mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices)
|
||||
mask_channel_length: 10 # Length of masks for features (channels)
|
||||
no_mask_channel_overlap: False # Whether to allow channel masks to overlap
|
||||
mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled)
|
||||
dropout_input: 0.1 # Dropout applied to input raw features
|
||||
dropout_features: 0.1 # Dropout applied to the features generator by convolutions
|
||||
final_dim: 0 # Project final representations and targets to this dimension
|
||||
n_negatives: 100 # Number of negatives to sample from the same audio sample
|
||||
cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch
|
||||
codebook_negatives: 0 # Number of negative examples in codebook
|
||||
negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states
|
||||
logit_temp: 0.1 # Temperature to divide logits by
|
||||
target_glu: False # Adds project and applies GLU to targets
|
||||
feature_grad_mult: 0.1 # Multiply extracted feature gradients
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: True
|
||||
num_workers: 4
|
||||
labels: [ ] # No vocab needed for pre-training
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 1
|
||||
labels: [ ]
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 4
|
||||
labels: [ ]
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 0.0005
|
||||
eps: 1e-06
|
||||
# optimizer arguments
|
||||
betas: [ 0.9, 0.98 ]
|
||||
weight_decay: 0.01
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: PolynomialDecayAnnealing
|
||||
min_lr: 0.000
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.1
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
num_nodes: 1
|
||||
max_epochs: 100
|
||||
max_steps: null # computed at runtime if not set
|
||||
val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
gradient_clip_val: 0.0
|
||||
amp_level: O0 # O1/O2 for mixed precision
|
||||
precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
log_every_n_steps: 10 # Interval of logging.
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
|
||||
sync_batchnorm: true
|
||||
|
||||
checkpoint_callback: false # Provided by exp_manager
|
||||
logger: false # Provided by exp_manager
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: true
|
||||
create_checkpoint_callback: true
|
||||
create_wandb_logger: false
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
@@ -0,0 +1,123 @@
|
||||
name: &name "Wav2vec_large_Pretrain"
|
||||
# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py
|
||||
# These parameters are based off the FairSeq implementation.
|
||||
# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml
|
||||
model:
|
||||
transformer_encoder:
|
||||
dropout: 0.0 # Dropout probability for the transformer
|
||||
conv: # Config for convolutional model that generates positional embeddings required for attention layer
|
||||
conv_pos: 128 # Number of filters for convolutional positional embeddings
|
||||
conv_pos_groups: 16 # Number of groups for convolutional positional embeddings
|
||||
encoder:
|
||||
encoder_layers: 24 # Number of encoder layers in transformer model
|
||||
encoder_layerdrop: 0.0 # Probability of dropping transformer layers
|
||||
embedding_dim: 1024 # Encoder embedding dim
|
||||
ffn_embedding_dim: 4096 # Encoder embedding dim for feed forward
|
||||
num_attention_heads: 16 # Number of encoder attention heads
|
||||
dropout: 0.0 # Dropout probability for transformer encoder
|
||||
activation_fn: gelu # Activation for transformer
|
||||
layer_norm_first: True # Apply layer norm first within the transformer
|
||||
loss:
|
||||
prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss
|
||||
feature_loss_weight: 0 # Weight applied to feature L2 Norm
|
||||
quantizer:
|
||||
quantize_targets: True # Quantize targets
|
||||
quantize_input: False # Quantize inputs
|
||||
same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled)
|
||||
latent_vars: 320 # Number of latent variables in each group of the codebook
|
||||
latent_groups: 2 # Number of groups within the codebook
|
||||
latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups
|
||||
latent_temp: [2, 0.1, 0.999995] # Quantize temperature (start, stop, decay factor)
|
||||
conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features
|
||||
extractor_mode: layer_norm # Mode for feature extractor. [default, layer_norm]
|
||||
conv_bias: True # Include bias in convolution feature extractor model
|
||||
masking:
|
||||
mask_prob: 0.65 # Probability of replacing token with mask
|
||||
mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson]
|
||||
mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices)
|
||||
mask_length: 10 # Length of mask when masking time steps
|
||||
no_mask_overlap: False # Whether to allow masks to overlap
|
||||
mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled)
|
||||
mask_channel_prob: 0 # Probability of replacing a feature with 0
|
||||
mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson]
|
||||
mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices)
|
||||
mask_channel_length: 10 # Length of masks for features (channels)
|
||||
no_mask_channel_overlap: False # Whether to allow channel masks to overlap
|
||||
mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled)
|
||||
dropout_input: 0.0 # Dropout applied to input raw features
|
||||
dropout_features: 0.0 # Dropout applied to the features generator by convolutions
|
||||
final_dim: 768 # Project final representations and targets to this dimension
|
||||
n_negatives: 100 # Number of negatives to sample from the same audio sample
|
||||
cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch
|
||||
codebook_negatives: 0 # Number of negative examples in codebook
|
||||
negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states
|
||||
logit_temp: 0.1 # Temperature to divide logits by
|
||||
target_glu: False # Adds project and applies GLU to targets
|
||||
feature_grad_mult: 1.0 # Multiply extracted feature gradients
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: True
|
||||
num_workers: 4
|
||||
labels: [ ] # No vocab needed for pre-training
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 1
|
||||
labels: [ ]
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 4
|
||||
labels: [ ]
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 0.0005
|
||||
eps: 1e-06
|
||||
# optimizer arguments
|
||||
betas: [ 0.9, 0.98 ]
|
||||
weight_decay: 0.01
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: PolynomialDecayAnnealing
|
||||
min_lr: 0.000
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.1
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
num_nodes: 1
|
||||
max_epochs: 100
|
||||
max_steps: null # computed at runtime if not set
|
||||
val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
gradient_clip_val: 0.0
|
||||
amp_level: O0 # O1/O2 for mixed precision
|
||||
precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
log_every_n_steps: 10 # Interval of logging.
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
|
||||
sync_batchnorm: true
|
||||
|
||||
checkpoint_callback: false # Provided by exp_manager
|
||||
logger: false # Provided by exp_manager
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: true
|
||||
create_checkpoint_callback: true
|
||||
create_wandb_logger: false
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
@@ -0,0 +1,97 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from nemo.collections.asr.models.wav2vec.wav2vec_model import Wav2VecEncoderModel
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils import logging
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
"""
|
||||
Pre-train a wav2vec 2.0 transformer model on audio. Uses a contrastive loss function to pre-train on unlabelled audio,
|
||||
using a task similar to masked language modeling in NLP. In wav2vec, we mask portions of the audio
|
||||
and the model is trained by minimising the distance of the ground truth for the masked section,
|
||||
using the ground truth quantized codebook representation. Distractors are obtained from other time steps.
|
||||
See :class:`Wav2VecCriterion` for more information.
|
||||
|
||||
Reference: https://arxiv.org/abs/2006.11477
|
||||
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=100
|
||||
|
||||
Basic run (on CPU for 50 epochs):
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=50
|
||||
|
||||
Using wav2vec-large with mixed precision:
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
--config-name=wav2vec_pretrain_large \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=100 \
|
||||
trainer.precision=16
|
||||
|
||||
Add PyTorch Lightning Trainer arguments from CLI:
|
||||
python wav2vec.py \
|
||||
... \
|
||||
+trainer.fast_dev_run=true
|
||||
|
||||
Hydra logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/.hydra)"
|
||||
PTL logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/lightning_logs)"
|
||||
|
||||
Override some args of optimizer:
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=2 \
|
||||
model.optim.args.params.betas=[0.8,0.5] \
|
||||
model.optim.args.params.weight_decay=0.0001
|
||||
|
||||
Override optimizer entirely
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=2 \
|
||||
~model.optim.args \
|
||||
+model.optim.args.betas=[0.8,0.5]\
|
||||
+model.optim.args.weight_decay=0.0005
|
||||
|
||||
"""
|
||||
|
||||
|
||||
@hydra_runner(config_path="configs", config_name="wav2vec_pretrain")
|
||||
def main(cfg: DictConfig):
|
||||
logging.info("Application config\n" + OmegaConf.to_yaml(cfg))
|
||||
|
||||
trainer = pl.Trainer(**cfg.trainer)
|
||||
exp_manager(trainer, cfg.get("exp_manager", None))
|
||||
|
||||
wav2vec_encoder_model = Wav2VecEncoderModel(cfg=cfg.model, trainer=trainer)
|
||||
trainer.fit(wav2vec_encoder_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
Reference in New Issue
Block a user