mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-18 07:41:03 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,201 @@
|
||||
# It contains the default values for training a Conformer ASR model medium size with RNNT loss and BPE-based vocabulary.
|
||||
# Default learning parameters in this config are set for effective batch size of 2K.
|
||||
# To train it with smaller batch sizes, you may need to re-tune the learning parameters or use higher accumulate_grad_batches.
|
||||
|
||||
# You may find more info about Conformer-CTC here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/models.html#conformer-ctc
|
||||
# Pre-trained models of Conformer-CTC can be found here: https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/asr/results.html
|
||||
|
||||
name: "Conformer-RNNT-BPE"
|
||||
|
||||
model:
|
||||
sample_rate: &sample_rate 16000
|
||||
compute_eval_loss: false # disables loss evaluation for validation and test data
|
||||
log_prediction: true # enables logging sample predictions in the output during training
|
||||
ctc_reduction: 'mean_batch'
|
||||
|
||||
model_defaults:
|
||||
se: true
|
||||
se_context_size: -1
|
||||
# encoder / decoder / joint values
|
||||
enc_hidden: ${model.encoder.d_model}
|
||||
pred_hidden: 640
|
||||
joint_hidden: 640
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 16 # you may increase batch_size if your memory allows
|
||||
shuffle: true
|
||||
num_workers: 8
|
||||
pin_memory: false
|
||||
use_start_end_token: true
|
||||
trim_silence: false
|
||||
max_duration: 16.7 # it is set for LibriSpeech, you may need to update it for your dataset
|
||||
min_duration: 0.1
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 16
|
||||
shuffle: false
|
||||
num_workers: 8
|
||||
pin_memory: false
|
||||
use_start_end_token: true
|
||||
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 16
|
||||
shuffle: false
|
||||
num_workers: 8
|
||||
pin_memory: false
|
||||
use_start_end_token: true
|
||||
|
||||
# you may find more detail on how to train a tokenizer at: /scripts/tokenizers/process_asr_text_tokenizer.py
|
||||
tokenizer:
|
||||
dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe)
|
||||
type: wpe # Can be either bpe (SentencePiece tokenizer) or wpe (WordPiece tokenizer)
|
||||
|
||||
preprocessor:
|
||||
_target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
sample_rate: *sample_rate
|
||||
normalize: "per_feature"
|
||||
window_size: 0.025
|
||||
window_stride: 0.01
|
||||
window: "hann"
|
||||
features: 80
|
||||
n_fft: 512
|
||||
frame_splicing: 1
|
||||
dither: 0.00001
|
||||
pad_to: 0
|
||||
|
||||
spec_augment:
|
||||
_target_: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
freq_masks: 2 # set to zero to disable it
|
||||
time_masks: 10 # set to zero to disable it
|
||||
freq_width: 27
|
||||
time_width: 0.05
|
||||
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConformerEncoder
|
||||
feat_in: ${model.preprocessor.features}
|
||||
feat_out: -1 # you may set it if you need different output size other than the default d_model
|
||||
n_layers: 16
|
||||
d_model: 256
|
||||
|
||||
# Sub-sampling params
|
||||
subsampling: striding # vggnet or striding
|
||||
subsampling_factor: 4 # must be power of 2
|
||||
subsampling_conv_channels: -1 # set to -1 to make it equal to the d_model
|
||||
|
||||
# Feed forward module's params
|
||||
ff_expansion_factor: 4
|
||||
|
||||
# Multi-headed Attention Module's params
|
||||
self_attention_model: rel_pos # rel_pos or abs_pos
|
||||
n_heads: 4
|
||||
xscaling: true # scales up the inputs by sqrt(d_model)
|
||||
untie_biases: true # unties the biases of the TransformerXL layers
|
||||
pos_emb_max_len: 5000
|
||||
|
||||
# Convolution module's params
|
||||
conv_kernel_size: 31
|
||||
|
||||
### regularization
|
||||
dropout: 0.1 # The dropout used in most of the Conformer Modules
|
||||
dropout_emb: 0.0 # The dropout used for embeddings
|
||||
dropout_att: 0.1 # The dropout for multi-headed attention modules
|
||||
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.RNNTDecoder
|
||||
normalization_mode: null
|
||||
random_state_sampling: false
|
||||
blank_as_pad: true
|
||||
|
||||
prednet:
|
||||
pred_hidden: ${model.model_defaults.pred_hidden}
|
||||
pred_rnn_layers: 1
|
||||
t_max: null
|
||||
dropout: 0.1
|
||||
|
||||
joint:
|
||||
_target_: nemo.collections.asr.modules.RNNTJoint
|
||||
log_softmax: null # 'null' would set it automatically according to CPU/GPU device
|
||||
|
||||
# fused mode
|
||||
experimental_fuse_loss_wer: true
|
||||
fused_batch_size: 16
|
||||
|
||||
jointnet:
|
||||
joint_hidden: ${model.model_defaults.joint_hidden}
|
||||
activation: "relu"
|
||||
dropout: 0.1
|
||||
|
||||
decoding:
|
||||
strategy: "greedy_batch"
|
||||
|
||||
# greedy strategy config
|
||||
greedy:
|
||||
max_symbols: 30
|
||||
|
||||
# beam strategy config
|
||||
beam:
|
||||
beam_size: 2
|
||||
score_norm: true
|
||||
tsd_max_sym_exp: 50 # for Time Synchronous Decoding
|
||||
alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding
|
||||
|
||||
optim:
|
||||
name: adam
|
||||
lr: 5.0
|
||||
# optimizer arguments
|
||||
betas: [0.9, 0.98]
|
||||
weight_decay: 1e-6
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: NoamAnnealing
|
||||
d_model: ${model.encoder.d_model}
|
||||
# scheduler config override
|
||||
warmup_steps: 10000
|
||||
warmup_ratio: null
|
||||
min_lr: 1e-6
|
||||
|
||||
trainer:
|
||||
gpus: -1 # number of GPUs, -1 would use all available GPUs
|
||||
num_nodes: 1
|
||||
max_epochs: 1000
|
||||
max_steps: null # computed at runtime if not set
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
gradient_clip_val: 0.0
|
||||
amp_level: O0 # O1/O2 for mixed precision
|
||||
precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
log_every_n_steps: 10 # Interval of logging.
|
||||
progress_bar_refresh_rate: 10
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
|
||||
sync_batchnorm: true
|
||||
checkpoint_callback: false # Provided by exp_manager
|
||||
logger: false # Provided by exp_manager
|
||||
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: ${name}
|
||||
create_tensorboard_logger: true
|
||||
create_checkpoint_callback: true
|
||||
checkpoint_callback_params:
|
||||
# in case of multiple validation sets, first one is used
|
||||
monitor: "val_wer"
|
||||
mode: "min"
|
||||
save_top_k: 3
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
|
||||
create_wandb_logger: false
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
@@ -0,0 +1,237 @@
|
||||
name: &name "ConvRNNT5x1"
|
||||
|
||||
model:
|
||||
sample_rate: 16000
|
||||
compute_eval_loss: true
|
||||
|
||||
labels: [ " ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'" ]
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
trim_silence: true
|
||||
max_duration: 16.7
|
||||
labels: ${model.labels}
|
||||
shuffle: true
|
||||
is_tarred: false
|
||||
tarred_audio_filepaths: null
|
||||
tarred_shard_strategy: "scatter"
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
shuffle: false
|
||||
labels: ${model.labels}
|
||||
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
shuffle: false
|
||||
labels: ${model.labels}
|
||||
|
||||
model_defaults:
|
||||
repeat: 5
|
||||
dropout: 0.0
|
||||
separable: true
|
||||
se: true
|
||||
se_context_size: -1
|
||||
# encoder / decoder / joint values
|
||||
enc_hidden: 1024
|
||||
pred_hidden: 320
|
||||
joint_hidden: 320
|
||||
|
||||
preprocessor:
|
||||
_target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
normalize: "per_feature"
|
||||
window_size: 0.02
|
||||
sample_rate: ${model.sample_rate}
|
||||
window_stride: 0.01
|
||||
window: "hann"
|
||||
features: &n_mels 80
|
||||
n_fft: 512
|
||||
frame_splicing: 1
|
||||
dither: 0.00001
|
||||
|
||||
spec_augment:
|
||||
_target_: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
rect_freq: 50
|
||||
rect_masks: 5
|
||||
rect_time: 120
|
||||
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASREncoder
|
||||
feat_in: *n_mels
|
||||
activation: relu
|
||||
conv_mask: true
|
||||
|
||||
jasper:
|
||||
- filters: 128
|
||||
repeat: 1
|
||||
kernel: [11]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [13]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [15]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [17]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [19]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: 1
|
||||
kernel: [21]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: ${model.model_defaults.enc_hidden}
|
||||
repeat: 1
|
||||
kernel: [1]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.RNNTDecoder
|
||||
normalization_mode: null
|
||||
random_state_sampling: false
|
||||
blank_as_pad: true
|
||||
|
||||
prednet:
|
||||
pred_hidden: ${model.model_defaults.pred_hidden}
|
||||
pred_rnn_layers: 1
|
||||
t_max: null
|
||||
dropout: 0.0
|
||||
|
||||
joint:
|
||||
_target_: nemo.collections.asr.modules.RNNTJoint
|
||||
log_softmax: null # sets it according to cpu/gpu device
|
||||
|
||||
# fused mode
|
||||
experimental_fuse_loss_wer: false
|
||||
fused_batch_size: 1
|
||||
|
||||
jointnet:
|
||||
joint_hidden: ${model.model_defaults.joint_hidden}
|
||||
activation: "relu"
|
||||
dropout: 0.0
|
||||
|
||||
decoding:
|
||||
strategy: "greedy_batch"
|
||||
|
||||
# greedy strategy config
|
||||
greedy:
|
||||
max_symbols: 30
|
||||
|
||||
# beam strategy config
|
||||
beam:
|
||||
beam_size: 2
|
||||
score_norm: true
|
||||
tsd_max_sym_exp: 50 # for Time Synchronous Decoding
|
||||
alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding
|
||||
|
||||
optim:
|
||||
name: adam
|
||||
# _target_: nemo.core.optim.optimizers.Adam
|
||||
lr: .1
|
||||
|
||||
# optimizer arguments
|
||||
betas: [0.9, 0.999]
|
||||
weight_decay: 0.0001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# scheduler config override
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.05
|
||||
min_lr: 1e-6
|
||||
last_epoch: -1
|
||||
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
max_epochs: 5
|
||||
max_steps: null # computed at runtime if not set
|
||||
num_nodes: 1
|
||||
accelerator: ddp
|
||||
precision: 32
|
||||
accumulate_grad_batches: 1
|
||||
checkpoint_callback: False # Provided by exp_manager
|
||||
logger: False # Provided by exp_manager
|
||||
log_every_n_steps: 1 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: True
|
||||
create_checkpoint_callback: True
|
||||
checkpoint_callback_params:
|
||||
monitor: "val_wer"
|
||||
mode: "min"
|
||||
create_wandb_logger: False
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
@@ -0,0 +1,238 @@
|
||||
name: &name "ConvRNNTBPE5x1"
|
||||
|
||||
model:
|
||||
sample_rate: 16000
|
||||
compute_eval_loss: true
|
||||
|
||||
tokenizer:
|
||||
dir: ??? # path to directory which contains either tokenizer.model (bpe) or vocab.txt (for wpe)
|
||||
type: ??? # Can be either bpe or wpe
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
trim_silence: true
|
||||
max_duration: 16.7
|
||||
labels: []
|
||||
shuffle: true
|
||||
is_tarred: false
|
||||
tarred_audio_filepaths: null
|
||||
tarred_shard_strategy: "scatter"
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
shuffle: false
|
||||
labels: []
|
||||
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: ${model.sample_rate}
|
||||
batch_size: 32
|
||||
shuffle: false
|
||||
labels: []
|
||||
|
||||
model_defaults:
|
||||
repeat: 5
|
||||
dropout: 0.0
|
||||
separable: true
|
||||
se: true
|
||||
se_context_size: -1
|
||||
# encoder / decoder / joint values
|
||||
enc_hidden: 1024
|
||||
pred_hidden: 320
|
||||
joint_hidden: 320
|
||||
|
||||
preprocessor:
|
||||
_target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
normalize: "per_feature"
|
||||
window_size: 0.02
|
||||
sample_rate: ${model.sample_rate}
|
||||
window_stride: 0.01
|
||||
window: "hann"
|
||||
features: &n_mels 80
|
||||
n_fft: 512
|
||||
frame_splicing: 1
|
||||
dither: 0.00001
|
||||
|
||||
spec_augment:
|
||||
_target_: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
rect_freq: 50
|
||||
rect_masks: 5
|
||||
rect_time: 120
|
||||
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASREncoder
|
||||
feat_in: *n_mels
|
||||
activation: relu
|
||||
conv_mask: true
|
||||
|
||||
jasper:
|
||||
- filters: 128
|
||||
repeat: 1
|
||||
kernel: [11]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [13]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [15]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [17]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: 256
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [19]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
- filters: 256
|
||||
repeat: 1
|
||||
kernel: [21]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
- filters: ${model.model_defaults.enc_hidden}
|
||||
repeat: 1
|
||||
kernel: [1]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.RNNTDecoder
|
||||
normalization_mode: null
|
||||
random_state_sampling: false
|
||||
blank_as_pad: true
|
||||
|
||||
prednet:
|
||||
pred_hidden: ${model.model_defaults.pred_hidden}
|
||||
pred_rnn_layers: 1
|
||||
t_max: null
|
||||
dropout: 0.0
|
||||
|
||||
joint:
|
||||
_target_: nemo.collections.asr.modules.RNNTJoint
|
||||
log_softmax: null # sets it according to cpu/gpu device
|
||||
|
||||
# fused mode
|
||||
experimental_fuse_loss_wer: false
|
||||
fused_batch_size: 1
|
||||
|
||||
jointnet:
|
||||
joint_hidden: ${model.model_defaults.joint_hidden}
|
||||
activation: "relu"
|
||||
dropout: 0.0
|
||||
|
||||
decoding:
|
||||
strategy: "greedy_batch"
|
||||
|
||||
# greedy strategy config
|
||||
greedy:
|
||||
max_symbols: 30
|
||||
|
||||
# beam strategy config
|
||||
beam:
|
||||
beam_size: 2
|
||||
score_norm: true
|
||||
tsd_max_sym_exp: 50 # for Time Synchronous Decoding
|
||||
alsd_max_target_len: 2.0 # for Alignment-Length Synchronous Decoding
|
||||
|
||||
optim:
|
||||
name: adam
|
||||
# _target_: nemo.core.optim.optimizers.Adam
|
||||
lr: .1
|
||||
|
||||
# optimizer arguments
|
||||
betas: [0.9, 0.999]
|
||||
weight_decay: 0.0001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# scheduler config override
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.05
|
||||
min_lr: 1e-6
|
||||
last_epoch: -1
|
||||
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
max_epochs: 5
|
||||
max_steps: null # computed at runtime if not set
|
||||
num_nodes: 1
|
||||
accelerator: ddp
|
||||
precision: 32
|
||||
accumulate_grad_batches: 1
|
||||
checkpoint_callback: False # Provided by exp_manager
|
||||
logger: False # Provided by exp_manager
|
||||
log_every_n_steps: 1 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: True
|
||||
create_checkpoint_callback: True
|
||||
checkpoint_callback_params:
|
||||
monitor: "val_wer"
|
||||
mode: "min"
|
||||
create_wandb_logger: False
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
@@ -0,0 +1,290 @@
|
||||
name: &name "QuartzNet15x5"
|
||||
|
||||
model:
|
||||
sample_rate: &sample_rate 16000
|
||||
repeat: &repeat 5
|
||||
dropout: &dropout 0.0
|
||||
separable: &separable true
|
||||
labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: *labels
|
||||
batch_size: 32
|
||||
trim_silence: True
|
||||
max_duration: 16.7
|
||||
shuffle: True
|
||||
is_tarred: False
|
||||
tarred_audio_filepaths: null
|
||||
tarred_shard_strategy: "scatter"
|
||||
augmentor:
|
||||
rir_noise_aug:
|
||||
prob: 0.5
|
||||
rir_manifest_path: ???
|
||||
rir_tar_filepaths: ???
|
||||
rir_prob: 0.5
|
||||
noise_manifest_paths: ???
|
||||
noise_tar_filepaths: ???
|
||||
min_snr_db: [0,0]
|
||||
max_snr_db: [30,30]
|
||||
orig_sample_rate: ???
|
||||
bg_noise_manifest_paths: ???
|
||||
bg_noise_tar_filepaths: ???
|
||||
bg_min_snr_db: [10,10]
|
||||
bg_max_snr_db: [40,40]
|
||||
bg_orig_sample_rate: ???
|
||||
transcode_aug:
|
||||
prob: 0.1
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: *labels
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
|
||||
preprocessor:
|
||||
_target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
normalize: "per_feature"
|
||||
window_size: 0.02
|
||||
sample_rate: *sample_rate
|
||||
window_stride: 0.01
|
||||
window: "hann"
|
||||
features: &n_mels 64
|
||||
n_fft: 512
|
||||
frame_splicing: 1
|
||||
dither: 0.00001
|
||||
stft_conv: false
|
||||
|
||||
spec_augment:
|
||||
_target_: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
rect_freq: 50
|
||||
rect_masks: 5
|
||||
rect_time: 120
|
||||
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASREncoder
|
||||
feat_in: *n_mels
|
||||
activation: relu
|
||||
conv_mask: true
|
||||
|
||||
jasper:
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [2]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [2]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [87]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: &enc_filters 1024
|
||||
kernel: [1]
|
||||
repeat: 1
|
||||
residual: false
|
||||
stride: [1]
|
||||
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASRDecoder
|
||||
feat_in: *enc_filters
|
||||
num_classes: 28
|
||||
vocabulary: *labels
|
||||
|
||||
optim:
|
||||
name: novograd
|
||||
# _target_: nemo.core.optim.optimizers.Novograd
|
||||
lr: .01
|
||||
# optimizer arguments
|
||||
betas: [0.8, 0.5]
|
||||
weight_decay: 0.001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# pytorch lightning args
|
||||
# monitor: val_loss
|
||||
# reduce_on_plateau: false
|
||||
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: null
|
||||
min_lr: 0.0
|
||||
last_epoch: -1
|
||||
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
max_epochs: 5
|
||||
max_steps: null # computed at runtime if not set
|
||||
num_nodes: 1
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
checkpoint_callback: False # Provided by exp_manager
|
||||
logger: False # Provided by exp_manager
|
||||
log_every_n_steps: 1 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: True
|
||||
create_checkpoint_callback: True
|
||||
create_wandb_logger: False
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
|
||||
hydra:
|
||||
run:
|
||||
dir: .
|
||||
job_logging:
|
||||
root:
|
||||
handlers: null
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script is based on speech_to_text_infer.py and allows you to score the hypotheses
|
||||
with sclite. A local installation from https://github.com/usnistgov/SCTK is required.
|
||||
Hypotheses and references are first saved in trn format and are scored after applying a glm
|
||||
file (if provided).
|
||||
"""
|
||||
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from argparse import ArgumentParser
|
||||
|
||||
import torch
|
||||
|
||||
from nemo.collections.asr.metrics.wer import WER
|
||||
from nemo.collections.asr.models import EncDecCTCModel
|
||||
from nemo.utils import logging
|
||||
|
||||
try:
|
||||
from torch.cuda.amp import autocast
|
||||
except ImportError:
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def autocast(enabled=None):
|
||||
yield
|
||||
|
||||
|
||||
def score_with_sctk(sctk_dir, ref_fname, hyp_fname, out_dir, glm=""):
|
||||
sclite_path = os.path.join(sctk_dir, "bin", "sclite")
|
||||
if not os.path.exists(sclite_path):
|
||||
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), sclite_path)
|
||||
# apply glm
|
||||
if os.path.exists(glm):
|
||||
rfilter_path = os.path.join(sctk_dir, "bin", "rfilter1")
|
||||
if not os.path.exists(rfilter_path):
|
||||
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), rfilter_path)
|
||||
hypglm = os.path.join(out_dir, os.path.basename(hyp_fname)) + ".glm"
|
||||
rfilt_cmd = [rfilter_path] + [glm]
|
||||
with open(hypglm, "w") as hypf, open(hyp_fname, "r") as hyp_in:
|
||||
subprocess.run(rfilt_cmd, stdin=hyp_in, stdout=hypf)
|
||||
refglm = os.path.join(out_dir, os.path.basename(ref_fname)) + ".glm"
|
||||
with open(refglm, "w") as reff, open(ref_fname, "r") as ref_in:
|
||||
subprocess.run(rfilt_cmd, stdin=ref_in, stdout=reff)
|
||||
else:
|
||||
refglm = ref_fname
|
||||
hypglm = hyp_fname
|
||||
|
||||
_ = subprocess.check_output(f"{sclite_path} -h {hypglm} -r {refglm} -i wsj -o all", shell=True)
|
||||
|
||||
|
||||
can_gpu = torch.cuda.is_available()
|
||||
|
||||
|
||||
def get_utt_info(manifest_path):
|
||||
info_list = []
|
||||
with open(manifest_path, "r") as utt_f:
|
||||
for line in utt_f:
|
||||
utt = json.loads(line)
|
||||
info_list.append(utt)
|
||||
|
||||
return info_list
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--asr_model", type=str, default="QuartzNet15x5Base-En", required=False, help="Pass: 'QuartzNet15x5Base-En'",
|
||||
)
|
||||
parser.add_argument("--dataset", type=str, required=True, help="path to evaluation data")
|
||||
parser.add_argument("--batch_size", type=int, default=4)
|
||||
parser.add_argument(
|
||||
"--dont_normalize_text",
|
||||
default=False,
|
||||
action='store_true',
|
||||
help="Turn off trasnscript normalization. Recommended for non-English.",
|
||||
)
|
||||
parser.add_argument("--out_dir", type=str, required=True, help="Destination dir for output files")
|
||||
parser.add_argument("--sctk_dir", type=str, required=False, default="", help="Path to sctk root dir")
|
||||
parser.add_argument("--glm", type=str, required=False, default="", help="Path to glm file")
|
||||
args = parser.parse_args()
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
if not os.path.exists(args.out_dir):
|
||||
os.makedirs(args.out_dir)
|
||||
|
||||
use_sctk = os.path.exists(args.sctk_dir)
|
||||
|
||||
if args.asr_model.endswith('.nemo'):
|
||||
logging.info(f"Using local ASR model from {args.asr_model}")
|
||||
asr_model = EncDecCTCModel.restore_from(restore_path=args.asr_model)
|
||||
else:
|
||||
logging.info(f"Using NGC cloud ASR model {args.asr_model}")
|
||||
asr_model = EncDecCTCModel.from_pretrained(model_name=args.asr_model)
|
||||
asr_model.setup_test_data(
|
||||
test_data_config={
|
||||
'sample_rate': 16000,
|
||||
'manifest_filepath': args.dataset,
|
||||
'labels': asr_model.decoder.vocabulary,
|
||||
'batch_size': args.batch_size,
|
||||
'normalize_transcripts': not args.dont_normalize_text,
|
||||
}
|
||||
)
|
||||
if can_gpu:
|
||||
asr_model = asr_model.cuda()
|
||||
asr_model.eval()
|
||||
labels_map = dict([(i, asr_model.decoder.vocabulary[i]) for i in range(len(asr_model.decoder.vocabulary))])
|
||||
|
||||
wer = WER(vocabulary=asr_model.decoder.vocabulary)
|
||||
hypotheses = []
|
||||
references = []
|
||||
all_log_probs = []
|
||||
for test_batch in asr_model.test_dataloader():
|
||||
if can_gpu:
|
||||
test_batch = [x.cuda() for x in test_batch]
|
||||
with autocast():
|
||||
log_probs, encoded_len, greedy_predictions = asr_model(
|
||||
input_signal=test_batch[0], input_signal_length=test_batch[1]
|
||||
)
|
||||
for r in log_probs.cpu().numpy():
|
||||
all_log_probs.append(r)
|
||||
hypotheses += wer.ctc_decoder_predictions_tensor(greedy_predictions)
|
||||
for batch_ind in range(greedy_predictions.shape[0]):
|
||||
reference = ''.join([labels_map[c] for c in test_batch[2][batch_ind].cpu().detach().numpy()])
|
||||
references.append(reference)
|
||||
del test_batch
|
||||
|
||||
info_list = get_utt_info(args.dataset)
|
||||
hypfile = os.path.join(args.out_dir, "hyp.trn")
|
||||
reffile = os.path.join(args.out_dir, "ref.trn")
|
||||
with open(hypfile, "w") as hyp_f, open(reffile, "w") as ref_f:
|
||||
for i in range(len(hypotheses)):
|
||||
utt_id = os.path.splitext(os.path.basename(info_list[i]['audio_filepath']))[0]
|
||||
# rfilter in sctk likes each transcript to have a space at the beginning
|
||||
hyp_f.write(" " + hypotheses[i] + " (" + utt_id + ")" + "\n")
|
||||
ref_f.write(" " + references[i] + " (" + utt_id + ")" + "\n")
|
||||
|
||||
if use_sctk:
|
||||
score_with_sctk(args.sctk_dir, reffile, hypfile, args.out_dir, glm=args.glm)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,235 @@
|
||||
name: &name "QuartzNet15x5"
|
||||
|
||||
model:
|
||||
sample_rate: &sample_rate 16000
|
||||
repeat: &repeat 5
|
||||
dropout: &dropout 0.0
|
||||
separable: &separable true
|
||||
labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: *sample_rate
|
||||
labels: *labels
|
||||
batch_size: 32
|
||||
max_duration: 16.7
|
||||
is_tarred: False
|
||||
tarred_audio_filepaths: null
|
||||
|
||||
validation_ds:
|
||||
labels: *labels
|
||||
manifest_filepath: ???
|
||||
sample_rate: *sample_rate
|
||||
batch_size: 32
|
||||
|
||||
preprocessor:
|
||||
sample_rate: *sample_rate
|
||||
window_size: 0.02
|
||||
window_stride: 0.01
|
||||
features: &n_mels 64
|
||||
|
||||
spec_augment:
|
||||
rect_freq: 50
|
||||
rect_masks: 5
|
||||
rect_time: 120
|
||||
|
||||
encoder:
|
||||
feat_in: *n_mels
|
||||
activation: relu
|
||||
|
||||
jasper:
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [2]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [2]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [87]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: &enc_filters 1024
|
||||
kernel: [1]
|
||||
repeat: 1
|
||||
residual: false
|
||||
stride: [1]
|
||||
|
||||
decoder:
|
||||
feat_in: *enc_filters
|
||||
num_classes: 28
|
||||
vocabulary: *labels
|
||||
|
||||
optim:
|
||||
name: novograd
|
||||
lr: .01
|
||||
|
||||
# optimizer arguments
|
||||
betas: [0.8, 0.5]
|
||||
weight_decay: 0.001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: null
|
||||
min_lr: 0.0
|
||||
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
max_epochs: 5
|
||||
num_nodes: 1
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
log_every_n_steps: 1 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
|
||||
exp_manager:
|
||||
name: *name
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils.config_utils import update_model_config
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
"""
|
||||
python speech_to_text_hybrid.py \
|
||||
--config-path="conf/quartznet" \
|
||||
--config-name="quartznet_15x5" \
|
||||
model.train_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/train_manifest.json" \
|
||||
model.validation_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/test_manifest.json" \
|
||||
trainer.gpus=1
|
||||
"""
|
||||
|
||||
|
||||
@hydra_runner(config_path="conf/quartznet", config_name="quartznet_15x5")
|
||||
def main(cfg):
|
||||
# Generate default asr model config
|
||||
asr_model_config = configs.EncDecCTCModelConfig()
|
||||
|
||||
# Merge hydra updates with model config
|
||||
# `drop_missing_subconfig=True` is necessary here. Without it, while the data class will instantiate and be added
|
||||
# to the config, it contains test_ds.sample_rate = MISSING and test_ds.labels = MISSING.
|
||||
# This will raise a OmegaConf MissingMandatoryValue error when processing the dataloaders inside
|
||||
# model_utils.resolve_test_dataloaders(model=self) (used for multi data loader support).
|
||||
# In general, any operation that tries to use a DictConfig with MISSING in it will fail,
|
||||
# other than explicit update operations to change MISSING to some actual value.
|
||||
asr_model_config = update_model_config(asr_model_config, cfg, drop_missing_subconfigs=True)
|
||||
|
||||
# From here on out, its a general OmegaConf DictConfig, directly usable by our code.
|
||||
trainer = pl.Trainer(**asr_model_config.trainer)
|
||||
exp_manager(trainer, asr_model_config.get("exp_manager", None))
|
||||
asr_model = EncDecCTCModel(cfg=asr_model_config.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,146 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
"""
|
||||
python speech_to_text_structured.py
|
||||
"""
|
||||
|
||||
# Generate default asr model config
|
||||
cfg = configs.EncDecCTCModelConfig()
|
||||
|
||||
# set global values
|
||||
cfg.model.repeat = 5
|
||||
cfg.model.separable = True
|
||||
|
||||
# fmt: off
|
||||
LABELS = [
|
||||
" ", "a", "b", "c", "d", "e",
|
||||
"f", "g", "h", "i", "j", "k",
|
||||
"l", "m", "n", "o", "p", "q",
|
||||
"r", "s", "t", "u", "v", "w",
|
||||
"x", "y", "z", "'",
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
qn_15x5 = [
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=256,
|
||||
repeat=1,
|
||||
kernel=[33],
|
||||
stride=[2],
|
||||
separable=cfg.model.separable,
|
||||
dilation=[1],
|
||||
dropout=cfg.model.dropout,
|
||||
residual=False,
|
||||
),
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=256,
|
||||
repeat=1,
|
||||
kernel=[33],
|
||||
stride=[1],
|
||||
separable=cfg.model.separable,
|
||||
dilation=[1],
|
||||
dropout=cfg.model.dropout,
|
||||
residual=True,
|
||||
),
|
||||
# ... repeat 14 more times
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=1024, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=cfg.model.dropout, residual=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
# Update values
|
||||
# MODEL UPDATES
|
||||
cfg.name = "Mini QuartzNet"
|
||||
cfg.model.labels = LABELS
|
||||
|
||||
# train ds
|
||||
cfg.model.train_ds.manifest_filepath = "<path to train dataset>"
|
||||
cfg.model.train_ds.labels = LABELS
|
||||
cfg.model.train_ds.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# validation ds
|
||||
cfg.model.validation_ds.manifest_filepath = "<path to test dataset>"
|
||||
cfg.model.validation_ds.labels = LABELS
|
||||
cfg.model.validation_ds.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# del `test_ds` does not work!
|
||||
# Refer - https://stackoverflow.com/questions/58119758/how-to-remove-dataclass-attributes
|
||||
# Hydra/OmegaConf dont allow custom .asdict() methods either
|
||||
# For now, explicitly set parameters
|
||||
cfg.model.test_ds.sample_rate = cfg.model.sample_rate
|
||||
cfg.model.test_ds.labels = cfg.model.labels
|
||||
|
||||
# preprocessor
|
||||
cfg.model.preprocessor.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# spec aug
|
||||
cfg.model.spec_augment.rect_masks = 5
|
||||
cfg.model.spec_augment.rect_freq = 50
|
||||
cfg.model.spec_augment.rect_time = 120
|
||||
|
||||
# encoder
|
||||
cfg.model.encoder.feat_in = cfg.model.preprocessor.features
|
||||
cfg.model.encoder.activation = 'relu'
|
||||
cfg.model.encoder.jasper = qn_15x5
|
||||
|
||||
# decoder
|
||||
cfg.model.decoder.feat_in = qn_15x5[-1].filters
|
||||
cfg.model.decoder.num_classes = len(LABELS)
|
||||
cfg.model.decoder.vocabulary = LABELS
|
||||
|
||||
# optim
|
||||
cfg.model.optim.name = 'novograd'
|
||||
cfg.model.optim.lr = 0.01
|
||||
|
||||
# `betas` dont exist inside the base config,
|
||||
# so they cannot be added as such!
|
||||
# Same for `weight_decay`.
|
||||
cfg.model.optim.betas = [0.8, 0.5]
|
||||
cfg.model.optim.weight_decay = 0.001
|
||||
|
||||
# sched
|
||||
# As parameters such as warmup_steps and warmup_ratio
|
||||
# dont exist inside the shell config, these values are not added!
|
||||
cfg.model.optim.sched.name = "CosineAnnealing"
|
||||
cfg.model.optim.sched.warmup_steps = None
|
||||
cfg.model.optim.sched.warmup_ratio = 0.01
|
||||
|
||||
# Trainer config
|
||||
cfg.trainer.gpus = 1
|
||||
cfg.trainer.max_epochs = 5
|
||||
|
||||
# Exp Manager config
|
||||
cfg.exp_manager.name = cfg.name
|
||||
|
||||
# Note usage of asdict
|
||||
trainer = pl.Trainer(**asdict(cfg.trainer))
|
||||
exp_manager(trainer, asdict(cfg.exp_manager))
|
||||
asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,90 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.core.config import modelPT, optimizers, schedulers
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
"""
|
||||
python speech_to_text_structured_v2.py
|
||||
"""
|
||||
|
||||
# fmt: off
|
||||
LABELS = [
|
||||
" ", "a", "b", "c", "d", "e",
|
||||
"f", "g", "h", "i", "j", "k",
|
||||
"l", "m", "n", "o", "p", "q",
|
||||
"r", "s", "t", "u", "v", "w",
|
||||
"x", "y", "z", "'",
|
||||
]
|
||||
|
||||
optim_cfg = optimizers.NovogradParams(
|
||||
lr=0.01,
|
||||
betas=(0.8, 0.5),
|
||||
weight_decay=0.001
|
||||
)
|
||||
|
||||
sched_cfg = schedulers.CosineAnnealingParams(
|
||||
warmup_steps=None,
|
||||
warmup_ratio=None,
|
||||
min_lr=0.0,
|
||||
)
|
||||
# fmt: on
|
||||
|
||||
|
||||
def main():
|
||||
# NeMo Model config
|
||||
cfg = modelPT.NemoConfig(name='Custom QuartzNet')
|
||||
|
||||
# Generate default asr model config
|
||||
builder = configs.EncDecCTCModelConfigBuilder(name='quartznet_15x5')
|
||||
|
||||
# set model global values
|
||||
builder.set_labels(LABELS)
|
||||
builder.set_optim(cfg=optim_cfg, sched_cfg=sched_cfg)
|
||||
|
||||
model_cfg = builder.build()
|
||||
|
||||
# set the model config to the NeMo Model
|
||||
cfg.model = model_cfg
|
||||
|
||||
# Update values
|
||||
# MODEL UPDATES
|
||||
# train ds
|
||||
model_cfg.train_ds.manifest_filepath = ""
|
||||
|
||||
# validation ds
|
||||
model_cfg.validation_ds.manifest_filepath = ""
|
||||
|
||||
# Trainer config
|
||||
cfg.trainer.gpus = 1
|
||||
cfg.trainer.max_epochs = 5
|
||||
|
||||
# Exp Manager config
|
||||
cfg.exp_manager.name = cfg.name
|
||||
|
||||
# Note usage of asdict
|
||||
trainer = pl.Trainer(**asdict(cfg.trainer))
|
||||
exp_manager(trainer, asdict(cfg.exp_manager))
|
||||
asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,123 @@
|
||||
name: &name "Wav2vec_Pretrain"
|
||||
# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py
|
||||
# These parameters are based off the FairSeq implementation.
|
||||
# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_base_librispeech.yaml
|
||||
model:
|
||||
transformer_encoder:
|
||||
dropout: 0.1 # Dropout probability for the transformer
|
||||
conv: # Config for convolutional model that generates positional embeddings required for attention layer
|
||||
conv_pos: 128 # Number of filters for convolutional positional embeddings
|
||||
conv_pos_groups: 16 # Number of groups for convolutional positional embeddings
|
||||
encoder:
|
||||
encoder_layers: 12 # Number of encoder layers in transformer model
|
||||
encoder_layerdrop: 0.05 # Probability of dropping transformer layers
|
||||
embedding_dim: 768 # Encoder embedding dim
|
||||
ffn_embedding_dim: 3072 # Encoder embedding dim for feed forward
|
||||
num_attention_heads: 8 # Number of encoder attention heads
|
||||
dropout: 0.1 # Dropout probability for transformer encoder
|
||||
activation_fn: gelu # Activation for transformer
|
||||
layer_norm_first: False # Apply layer norm first within the transformer
|
||||
loss:
|
||||
prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss
|
||||
feature_loss_weight: 0 # Weight applied to feature L2 Norm
|
||||
quantizer:
|
||||
quantize_targets: True # Quantize targets
|
||||
quantize_input: False # Quantize inputs
|
||||
same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled)
|
||||
latent_vars: 320 # Number of latent variables in each group of the codebook
|
||||
latent_groups: 2 # Number of groups within the codebook
|
||||
latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups
|
||||
latent_temp: [2, 0.5, 0.999995] # Quantize temperature (start, stop, decay factor)
|
||||
conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features
|
||||
extractor_mode: default # Mode for feature extractor. [Default, layer_norm]
|
||||
conv_bias: False # Include bias in convolution feature extractor model
|
||||
masking:
|
||||
mask_prob: 0.65 # Probability of replacing token with mask
|
||||
mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson]
|
||||
mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices)
|
||||
mask_length: 10 # Length of mask when masking time steps
|
||||
no_mask_overlap: False # Whether to allow masks to overlap
|
||||
mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled)
|
||||
mask_channel_prob: 0 # Probability of replacing a feature with 0
|
||||
mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson]
|
||||
mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices)
|
||||
mask_channel_length: 10 # Length of masks for features (channels)
|
||||
no_mask_channel_overlap: False # Whether to allow channel masks to overlap
|
||||
mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled)
|
||||
dropout_input: 0.1 # Dropout applied to input raw features
|
||||
dropout_features: 0.1 # Dropout applied to the features generator by convolutions
|
||||
final_dim: 0 # Project final representations and targets to this dimension
|
||||
n_negatives: 100 # Number of negatives to sample from the same audio sample
|
||||
cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch
|
||||
codebook_negatives: 0 # Number of negative examples in codebook
|
||||
negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states
|
||||
logit_temp: 0.1 # Temperature to divide logits by
|
||||
target_glu: False # Adds project and applies GLU to targets
|
||||
feature_grad_mult: 0.1 # Multiply extracted feature gradients
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: True
|
||||
num_workers: 4
|
||||
labels: [ ] # No vocab needed for pre-training
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 1
|
||||
labels: [ ]
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 4
|
||||
labels: [ ]
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 0.0005
|
||||
eps: 1e-06
|
||||
# optimizer arguments
|
||||
betas: [ 0.9, 0.98 ]
|
||||
weight_decay: 0.01
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: PolynomialDecayAnnealing
|
||||
min_lr: 0.000
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.1
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
num_nodes: 1
|
||||
max_epochs: 100
|
||||
max_steps: null # computed at runtime if not set
|
||||
val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
gradient_clip_val: 0.0
|
||||
amp_level: O0 # O1/O2 for mixed precision
|
||||
precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
log_every_n_steps: 10 # Interval of logging.
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
|
||||
sync_batchnorm: true
|
||||
|
||||
checkpoint_callback: false # Provided by exp_manager
|
||||
logger: false # Provided by exp_manager
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: true
|
||||
create_checkpoint_callback: true
|
||||
create_wandb_logger: false
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
@@ -0,0 +1,123 @@
|
||||
name: &name "Wav2vec_large_Pretrain"
|
||||
# For more details on the model parameters, see nemo/collections/asr/models/wav2vec/wav2vec_config.py
|
||||
# These parameters are based off the FairSeq implementation.
|
||||
# See here: https://github.com/pytorch/fairseq/blob/master/examples/wav2vec/config/pretraining/wav2vec2_large_librivox.yaml
|
||||
model:
|
||||
transformer_encoder:
|
||||
dropout: 0.0 # Dropout probability for the transformer
|
||||
conv: # Config for convolutional model that generates positional embeddings required for attention layer
|
||||
conv_pos: 128 # Number of filters for convolutional positional embeddings
|
||||
conv_pos_groups: 16 # Number of groups for convolutional positional embeddings
|
||||
encoder:
|
||||
encoder_layers: 24 # Number of encoder layers in transformer model
|
||||
encoder_layerdrop: 0.0 # Probability of dropping transformer layers
|
||||
embedding_dim: 1024 # Encoder embedding dim
|
||||
ffn_embedding_dim: 4096 # Encoder embedding dim for feed forward
|
||||
num_attention_heads: 16 # Number of encoder attention heads
|
||||
dropout: 0.0 # Dropout probability for transformer encoder
|
||||
activation_fn: gelu # Activation for transformer
|
||||
layer_norm_first: True # Apply layer norm first within the transformer
|
||||
loss:
|
||||
prob_ppl_weight: 0.1 # Weight applied to quantized prob perplexity loss
|
||||
feature_loss_weight: 0 # Weight applied to feature L2 Norm
|
||||
quantizer:
|
||||
quantize_targets: True # Quantize targets
|
||||
quantize_input: False # Quantize inputs
|
||||
same_quantizer: False # Use the same quantizer for inputs and targets (If both enabled)
|
||||
latent_vars: 320 # Number of latent variables in each group of the codebook
|
||||
latent_groups: 2 # Number of groups within the codebook
|
||||
latent_dim: 0 # If greater than 0, use the dim for latest variables else inferred by final dim / latent groups
|
||||
latent_temp: [2, 0.1, 0.999995] # Quantize temperature (start, stop, decay factor)
|
||||
conv_feature_encoder: # Config for feature extractor taking raw audio waveform and converting to features
|
||||
extractor_mode: layer_norm # Mode for feature extractor. [default, layer_norm]
|
||||
conv_bias: True # Include bias in convolution feature extractor model
|
||||
masking:
|
||||
mask_prob: 0.65 # Probability of replacing token with mask
|
||||
mask_type: static # Used to select configuration to compute mask lengths [static, uniform, normal, poisson]
|
||||
mask_other: 0 # Secondary mask used for complex distributions (see help in compute_mask_indices)
|
||||
mask_length: 10 # Length of mask when masking time steps
|
||||
no_mask_overlap: False # Whether to allow masks to overlap
|
||||
mask_min_space: 1 # Minimum space beetween spans (if no overlap is enabled)
|
||||
mask_channel_prob: 0 # Probability of replacing a feature with 0
|
||||
mask_channel_type: static # Used to select configuration to compute mask channel lengths [static, uniform, normal, poisson]
|
||||
mask_channel_other: 0 # Secondary mask argument (used for more complex distributions (see help in compute_mask_indices)
|
||||
mask_channel_length: 10 # Length of masks for features (channels)
|
||||
no_mask_channel_overlap: False # Whether to allow channel masks to overlap
|
||||
mask_channel_min_space: 1 # Minimum space between spans (if no overlap is enabled)
|
||||
dropout_input: 0.0 # Dropout applied to input raw features
|
||||
dropout_features: 0.0 # Dropout applied to the features generator by convolutions
|
||||
final_dim: 768 # Project final representations and targets to this dimension
|
||||
n_negatives: 100 # Number of negatives to sample from the same audio sample
|
||||
cross_sample_negatives: 0 # Number of negatives to sample from any sample in the batch
|
||||
codebook_negatives: 0 # Number of negative examples in codebook
|
||||
negatives_from_everywhere: False # Sample negatives from everywhere, not just masked states
|
||||
logit_temp: 0.1 # Temperature to divide logits by
|
||||
target_glu: False # Adds project and applies GLU to targets
|
||||
feature_grad_mult: 1.0 # Multiply extracted feature gradients
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: True
|
||||
num_workers: 4
|
||||
labels: [ ] # No vocab needed for pre-training
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 1
|
||||
labels: [ ]
|
||||
test_ds:
|
||||
manifest_filepath: null
|
||||
sample_rate: 16000
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
num_workers: 4
|
||||
labels: [ ]
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 0.0005
|
||||
eps: 1e-06
|
||||
# optimizer arguments
|
||||
betas: [ 0.9, 0.98 ]
|
||||
weight_decay: 0.01
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: PolynomialDecayAnnealing
|
||||
min_lr: 0.000
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.1
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
num_nodes: 1
|
||||
max_epochs: 100
|
||||
max_steps: null # computed at runtime if not set
|
||||
val_check_interval: 0.5 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
gradient_clip_val: 0.0
|
||||
amp_level: O0 # O1/O2 for mixed precision
|
||||
precision: 32 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
log_every_n_steps: 10 # Interval of logging.
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
check_val_every_n_epoch: 1 # number of evaluations on validation every n epochs
|
||||
sync_batchnorm: true
|
||||
|
||||
checkpoint_callback: false # Provided by exp_manager
|
||||
logger: false # Provided by exp_manager
|
||||
exp_manager:
|
||||
exp_dir: null
|
||||
name: *name
|
||||
create_tensorboard_logger: true
|
||||
create_checkpoint_callback: true
|
||||
create_wandb_logger: false
|
||||
wandb_logger_kwargs:
|
||||
name: null
|
||||
project: null
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
@@ -0,0 +1,97 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from nemo.collections.asr.models.wav2vec.wav2vec_model import Wav2VecEncoderModel
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils import logging
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
"""
|
||||
Pre-train a wav2vec 2.0 transformer model on audio. Uses a contrastive loss function to pre-train on unlabelled audio,
|
||||
using a task similar to masked language modeling in NLP. In wav2vec, we mask portions of the audio
|
||||
and the model is trained by minimising the distance of the ground truth for the masked section,
|
||||
using the ground truth quantized codebook representation. Distractors are obtained from other time steps.
|
||||
See :class:`Wav2VecCriterion` for more information.
|
||||
|
||||
Reference: https://arxiv.org/abs/2006.11477
|
||||
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=100
|
||||
|
||||
Basic run (on CPU for 50 epochs):
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=50
|
||||
|
||||
Using wav2vec-large with mixed precision:
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
--config-name=wav2vec_pretrain_large \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=1 \
|
||||
trainer.max_epochs=100 \
|
||||
trainer.precision=16
|
||||
|
||||
Add PyTorch Lightning Trainer arguments from CLI:
|
||||
python wav2vec.py \
|
||||
... \
|
||||
+trainer.fast_dev_run=true
|
||||
|
||||
Hydra logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/.hydra)"
|
||||
PTL logs will be found in "$(./outputs/$(date +"%y-%m-%d")/$(date +"%H-%M-%S")/lightning_logs)"
|
||||
|
||||
Override some args of optimizer:
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=2 \
|
||||
model.optim.args.params.betas=[0.8,0.5] \
|
||||
model.optim.args.params.weight_decay=0.0001
|
||||
|
||||
Override optimizer entirely
|
||||
python examples/asr/experimental/wav2vec/wav2vec_pretrain.py \
|
||||
model.train_ds.manifest_filepath="./examples/asr/train.tsv" \
|
||||
model.validation_ds.manifest_filepath="./examples/asr/valid.tsv" \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=2 \
|
||||
~model.optim.args \
|
||||
+model.optim.args.betas=[0.8,0.5]\
|
||||
+model.optim.args.weight_decay=0.0005
|
||||
|
||||
"""
|
||||
|
||||
|
||||
@hydra_runner(config_path="configs", config_name="wav2vec_pretrain")
|
||||
def main(cfg: DictConfig):
|
||||
logging.info("Application config\n" + OmegaConf.to_yaml(cfg))
|
||||
|
||||
trainer = pl.Trainer(**cfg.trainer)
|
||||
exp_manager(trainer, cfg.get("exp_manager", None))
|
||||
|
||||
wav2vec_encoder_model = Wav2VecEncoderModel(cfg=cfg.model, trainer=trainer)
|
||||
trainer.fit(wav2vec_encoder_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
Reference in New Issue
Block a user