mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-18 00:33:15 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,235 @@
|
||||
name: &name "QuartzNet15x5"
|
||||
|
||||
model:
|
||||
sample_rate: &sample_rate 16000
|
||||
repeat: &repeat 5
|
||||
dropout: &dropout 0.0
|
||||
separable: &separable true
|
||||
labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: *sample_rate
|
||||
labels: *labels
|
||||
batch_size: 32
|
||||
max_duration: 16.7
|
||||
is_tarred: False
|
||||
tarred_audio_filepaths: null
|
||||
|
||||
validation_ds:
|
||||
labels: *labels
|
||||
manifest_filepath: ???
|
||||
sample_rate: *sample_rate
|
||||
batch_size: 32
|
||||
|
||||
preprocessor:
|
||||
sample_rate: *sample_rate
|
||||
window_size: 0.02
|
||||
window_stride: 0.01
|
||||
features: &n_mels 64
|
||||
|
||||
spec_augment:
|
||||
rect_freq: 50
|
||||
rect_masks: 5
|
||||
rect_time: 120
|
||||
|
||||
encoder:
|
||||
feat_in: *n_mels
|
||||
activation: relu
|
||||
|
||||
jasper:
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [2]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [39]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [51]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [63]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [2]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [87]
|
||||
repeat: 1
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: &enc_filters 1024
|
||||
kernel: [1]
|
||||
repeat: 1
|
||||
residual: false
|
||||
stride: [1]
|
||||
|
||||
decoder:
|
||||
feat_in: *enc_filters
|
||||
num_classes: 28
|
||||
vocabulary: *labels
|
||||
|
||||
optim:
|
||||
name: novograd
|
||||
lr: .01
|
||||
|
||||
# optimizer arguments
|
||||
betas: [0.8, 0.5]
|
||||
weight_decay: 0.001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: null
|
||||
min_lr: 0.0
|
||||
|
||||
trainer:
|
||||
gpus: 0 # number of gpus
|
||||
max_epochs: 5
|
||||
num_nodes: 1
|
||||
accelerator: ddp
|
||||
accumulate_grad_batches: 1
|
||||
log_every_n_steps: 1 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
|
||||
exp_manager:
|
||||
name: *name
|
||||
@@ -0,0 +1,56 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils.config_utils import update_model_config
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
"""
|
||||
python speech_to_text_hybrid.py \
|
||||
--config-path="conf/quartznet" \
|
||||
--config-name="quartznet_15x5" \
|
||||
model.train_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/train_manifest.json" \
|
||||
model.validation_ds.manifest_filepath="/home/smajumdar/PycharmProjects/NeMo-som/examples/asr/an4/test_manifest.json" \
|
||||
trainer.gpus=1
|
||||
"""
|
||||
|
||||
|
||||
@hydra_runner(config_path="conf/quartznet", config_name="quartznet_15x5")
|
||||
def main(cfg):
|
||||
# Generate default asr model config
|
||||
asr_model_config = configs.EncDecCTCModelConfig()
|
||||
|
||||
# Merge hydra updates with model config
|
||||
# `drop_missing_subconfig=True` is necessary here. Without it, while the data class will instantiate and be added
|
||||
# to the config, it contains test_ds.sample_rate = MISSING and test_ds.labels = MISSING.
|
||||
# This will raise a OmegaConf MissingMandatoryValue error when processing the dataloaders inside
|
||||
# model_utils.resolve_test_dataloaders(model=self) (used for multi data loader support).
|
||||
# In general, any operation that tries to use a DictConfig with MISSING in it will fail,
|
||||
# other than explicit update operations to change MISSING to some actual value.
|
||||
asr_model_config = update_model_config(asr_model_config, cfg, drop_missing_subconfigs=True)
|
||||
|
||||
# From here on out, its a general OmegaConf DictConfig, directly usable by our code.
|
||||
trainer = pl.Trainer(**asr_model_config.trainer)
|
||||
exp_manager(trainer, asr_model_config.get("exp_manager", None))
|
||||
asr_model = EncDecCTCModel(cfg=asr_model_config.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,146 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
"""
|
||||
python speech_to_text_structured.py
|
||||
"""
|
||||
|
||||
# Generate default asr model config
|
||||
cfg = configs.EncDecCTCModelConfig()
|
||||
|
||||
# set global values
|
||||
cfg.model.repeat = 5
|
||||
cfg.model.separable = True
|
||||
|
||||
# fmt: off
|
||||
LABELS = [
|
||||
" ", "a", "b", "c", "d", "e",
|
||||
"f", "g", "h", "i", "j", "k",
|
||||
"l", "m", "n", "o", "p", "q",
|
||||
"r", "s", "t", "u", "v", "w",
|
||||
"x", "y", "z", "'",
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
qn_15x5 = [
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=256,
|
||||
repeat=1,
|
||||
kernel=[33],
|
||||
stride=[2],
|
||||
separable=cfg.model.separable,
|
||||
dilation=[1],
|
||||
dropout=cfg.model.dropout,
|
||||
residual=False,
|
||||
),
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=256,
|
||||
repeat=1,
|
||||
kernel=[33],
|
||||
stride=[1],
|
||||
separable=cfg.model.separable,
|
||||
dilation=[1],
|
||||
dropout=cfg.model.dropout,
|
||||
residual=True,
|
||||
),
|
||||
# ... repeat 14 more times
|
||||
nemo_asr.modules.conv_asr.JasperEncoderConfig(
|
||||
filters=1024, repeat=1, kernel=[1], stride=[1], dilation=[1], dropout=cfg.model.dropout, residual=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
# Update values
|
||||
# MODEL UPDATES
|
||||
cfg.name = "Mini QuartzNet"
|
||||
cfg.model.labels = LABELS
|
||||
|
||||
# train ds
|
||||
cfg.model.train_ds.manifest_filepath = "<path to train dataset>"
|
||||
cfg.model.train_ds.labels = LABELS
|
||||
cfg.model.train_ds.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# validation ds
|
||||
cfg.model.validation_ds.manifest_filepath = "<path to test dataset>"
|
||||
cfg.model.validation_ds.labels = LABELS
|
||||
cfg.model.validation_ds.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# del `test_ds` does not work!
|
||||
# Refer - https://stackoverflow.com/questions/58119758/how-to-remove-dataclass-attributes
|
||||
# Hydra/OmegaConf dont allow custom .asdict() methods either
|
||||
# For now, explicitly set parameters
|
||||
cfg.model.test_ds.sample_rate = cfg.model.sample_rate
|
||||
cfg.model.test_ds.labels = cfg.model.labels
|
||||
|
||||
# preprocessor
|
||||
cfg.model.preprocessor.sample_rate = cfg.model.sample_rate
|
||||
|
||||
# spec aug
|
||||
cfg.model.spec_augment.rect_masks = 5
|
||||
cfg.model.spec_augment.rect_freq = 50
|
||||
cfg.model.spec_augment.rect_time = 120
|
||||
|
||||
# encoder
|
||||
cfg.model.encoder.feat_in = cfg.model.preprocessor.features
|
||||
cfg.model.encoder.activation = 'relu'
|
||||
cfg.model.encoder.jasper = qn_15x5
|
||||
|
||||
# decoder
|
||||
cfg.model.decoder.feat_in = qn_15x5[-1].filters
|
||||
cfg.model.decoder.num_classes = len(LABELS)
|
||||
cfg.model.decoder.vocabulary = LABELS
|
||||
|
||||
# optim
|
||||
cfg.model.optim.name = 'novograd'
|
||||
cfg.model.optim.lr = 0.01
|
||||
|
||||
# `betas` dont exist inside the base config,
|
||||
# so they cannot be added as such!
|
||||
# Same for `weight_decay`.
|
||||
cfg.model.optim.betas = [0.8, 0.5]
|
||||
cfg.model.optim.weight_decay = 0.001
|
||||
|
||||
# sched
|
||||
# As parameters such as warmup_steps and warmup_ratio
|
||||
# dont exist inside the shell config, these values are not added!
|
||||
cfg.model.optim.sched.name = "CosineAnnealing"
|
||||
cfg.model.optim.sched.warmup_steps = None
|
||||
cfg.model.optim.sched.warmup_ratio = 0.01
|
||||
|
||||
# Trainer config
|
||||
cfg.trainer.gpus = 1
|
||||
cfg.trainer.max_epochs = 5
|
||||
|
||||
# Exp Manager config
|
||||
cfg.exp_manager.name = cfg.name
|
||||
|
||||
# Note usage of asdict
|
||||
trainer = pl.Trainer(**asdict(cfg.trainer))
|
||||
exp_manager(trainer, asdict(cfg.exp_manager))
|
||||
asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
@@ -0,0 +1,90 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
from nemo.collections.asr.models import EncDecCTCModel, configs
|
||||
from nemo.core.config import modelPT, optimizers, schedulers
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
"""
|
||||
python speech_to_text_structured_v2.py
|
||||
"""
|
||||
|
||||
# fmt: off
|
||||
LABELS = [
|
||||
" ", "a", "b", "c", "d", "e",
|
||||
"f", "g", "h", "i", "j", "k",
|
||||
"l", "m", "n", "o", "p", "q",
|
||||
"r", "s", "t", "u", "v", "w",
|
||||
"x", "y", "z", "'",
|
||||
]
|
||||
|
||||
optim_cfg = optimizers.NovogradParams(
|
||||
lr=0.01,
|
||||
betas=(0.8, 0.5),
|
||||
weight_decay=0.001
|
||||
)
|
||||
|
||||
sched_cfg = schedulers.CosineAnnealingParams(
|
||||
warmup_steps=None,
|
||||
warmup_ratio=None,
|
||||
min_lr=0.0,
|
||||
)
|
||||
# fmt: on
|
||||
|
||||
|
||||
def main():
|
||||
# NeMo Model config
|
||||
cfg = modelPT.NemoConfig(name='Custom QuartzNet')
|
||||
|
||||
# Generate default asr model config
|
||||
builder = configs.EncDecCTCModelConfigBuilder(name='quartznet_15x5')
|
||||
|
||||
# set model global values
|
||||
builder.set_labels(LABELS)
|
||||
builder.set_optim(cfg=optim_cfg, sched_cfg=sched_cfg)
|
||||
|
||||
model_cfg = builder.build()
|
||||
|
||||
# set the model config to the NeMo Model
|
||||
cfg.model = model_cfg
|
||||
|
||||
# Update values
|
||||
# MODEL UPDATES
|
||||
# train ds
|
||||
model_cfg.train_ds.manifest_filepath = ""
|
||||
|
||||
# validation ds
|
||||
model_cfg.validation_ds.manifest_filepath = ""
|
||||
|
||||
# Trainer config
|
||||
cfg.trainer.gpus = 1
|
||||
cfg.trainer.max_epochs = 5
|
||||
|
||||
# Exp Manager config
|
||||
cfg.exp_manager.name = cfg.name
|
||||
|
||||
# Note usage of asdict
|
||||
trainer = pl.Trainer(**asdict(cfg.trainer))
|
||||
exp_manager(trainer, asdict(cfg.exp_manager))
|
||||
asr_model = EncDecCTCModel(cfg=cfg.model, trainer=trainer)
|
||||
|
||||
trainer.fit(asr_model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main() # noqa pylint: disable=no-value-for-parameter
|
||||
Reference in New Issue
Block a user