mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-16 00:13:07 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,199 @@
|
||||
|
||||
/* Width of template */
|
||||
|
||||
.wy-nav-content {
|
||||
max-width: 1200px !important;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* Standard Text Formatting */
|
||||
|
||||
h1
|
||||
{
|
||||
color: #76b900;
|
||||
text-align: center;
|
||||
background-color: #333333;
|
||||
}
|
||||
|
||||
h2
|
||||
{
|
||||
color: #ffffff;
|
||||
background-color: #76b900;
|
||||
Padding: 5px;
|
||||
}
|
||||
|
||||
h3
|
||||
{
|
||||
padding-top: 0px;
|
||||
border-top: solid 3px #76b900;
|
||||
border-bottom: solid 3px #76b900;
|
||||
}
|
||||
|
||||
p
|
||||
{
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
/* Link Colors */
|
||||
a
|
||||
{
|
||||
color: #76b900;
|
||||
}
|
||||
|
||||
a:visited
|
||||
{
|
||||
color: #218219;
|
||||
}
|
||||
|
||||
|
||||
/* ----------------------------------------------TABLES--------------------------------------- */
|
||||
|
||||
/* Table head Color */
|
||||
thead td
|
||||
{
|
||||
background-color: #333333 !important;
|
||||
}
|
||||
|
||||
.row-odd p
|
||||
{
|
||||
/*padding-bottom: 0px;*/
|
||||
/*margin-bottom: 0px;*/
|
||||
}
|
||||
/* even rows*/
|
||||
|
||||
.row-even tr
|
||||
{
|
||||
background-color: #e5f1e6 !important;
|
||||
}
|
||||
|
||||
/* odd rows*/
|
||||
|
||||
|
||||
.wy-table-responsive table tr {
|
||||
background-color: #ffffff !important;
|
||||
}
|
||||
|
||||
|
||||
|
||||
.wy-table-responsive table td {
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
|
||||
/* Removes bottom margin in tables*/
|
||||
|
||||
.rst-content .line-block {
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
.wy-table-responsive {
|
||||
overflow: visible !important;
|
||||
}
|
||||
|
||||
/* reduces the size of text in multiline table columns. */
|
||||
|
||||
.rst-content table.docutils td
|
||||
{
|
||||
font-size: 80%;
|
||||
}
|
||||
|
||||
.rst-content dl:not(.docutils) dt
|
||||
{
|
||||
|
||||
background-color: inherit;
|
||||
color: #000000;
|
||||
border-top: solid 0px #000000;
|
||||
|
||||
}
|
||||
|
||||
.rst-content dl:not(.docutils) dt:before {
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.rst-content .line-block {
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
.wy-side-nav-search, .wy-nav-top
|
||||
{
|
||||
background-color: #000000;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.wy-side-nav-search img
|
||||
{
|
||||
padding: 0px;
|
||||
padding: 0px 0px;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.wy-side-nav-search input[type=text]
|
||||
{
|
||||
border-radius: 0px;
|
||||
}
|
||||
|
||||
|
||||
.wy-menu-vertical p.caption
|
||||
{
|
||||
color: #76b900;
|
||||
}
|
||||
|
||||
|
||||
.wy-side-nav-search>a img.logo, .wy-side-nav-search .wy-dropdown>a img.logo
|
||||
{
|
||||
margin: 0px 0px 0px 0px;
|
||||
}
|
||||
|
||||
.wy-nav-content
|
||||
{
|
||||
margin: 0;
|
||||
min-height: 100%;
|
||||
height: 100%;
|
||||
background: #ffffff;
|
||||
}
|
||||
|
||||
/* List (numbered, bulleted) padding Fix */
|
||||
|
||||
|
||||
.wy-plain-list-decimal li
|
||||
{
|
||||
margin-top: -6px;
|
||||
margin-bottom: -6px;
|
||||
}
|
||||
|
||||
.rst-content .section ol.loweralpha
|
||||
{
|
||||
margin-top: -6px;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.wy-plain-list-disc, .rst-content .toctree-wrapper ul, article ul
|
||||
{
|
||||
margin-top: 0px !important;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
/* Alert Boxes */
|
||||
/* Background color of Alert Box Title */
|
||||
|
||||
.rst-content .section ul
|
||||
{
|
||||
margin-top: -12px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.wy-alert.wy-alert-info .wy-alert-title, .rst-content .note .wy-alert-title, .rst-content .wy-alert-info.attention .wy-alert-title, .rst-content .wy-alert-info.caution .wy-alert-title, .rst-content .wy-alert-info.danger .wy-alert-title, .rst-content .wy-alert-info.error .wy-alert-title, .rst-content .wy-alert-info.hint .wy-alert-title, .rst-content .wy-alert-info.important .wy-alert-title, .rst-content .wy-alert-info.tip .wy-alert-title, .rst-content .wy-alert-info.warning .wy-alert-title, .rst-content .seealso .wy-alert-title, .rst-content .wy-alert-info.admonition-todo .wy-alert-title, .rst-content .wy-alert-info.admonition .wy-alert-title, .wy-alert.wy-alert-info .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-info .admonition-title, .rst-content .note .admonition-title, .rst-content .wy-alert-info.attention .admonition-title, .rst-content .wy-alert-info.caution .admonition-title, .rst-content .wy-alert-info.danger .admonition-title, .rst-content .wy-alert-info.error .admonition-title, .rst-content .wy-alert-info.hint .admonition-title, .rst-content .wy-alert-info.important .admonition-title, .rst-content .wy-alert-info.tip .admonition-title, .rst-content .wy-alert-info.warning .admonition-title, .rst-content .seealso .admonition-title, .rst-content .wy-alert-info.admonition-todo .admonition-title, .rst-content .wy-alert-info.admonition .admonition-title
|
||||
{
|
||||
background: #76b900;
|
||||
}
|
||||
|
||||
/* Background and Font Color of Alert Box Main Body*/
|
||||
.wy-alert.wy-alert-info, .rst-content .note, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .rst-content .seealso, .rst-content .wy-alert-info.admonition-todo, .rst-content .wy-alert-info.admonition {
|
||||
background: #333333;
|
||||
color: #999999;
|
||||
}
|
||||
|
||||
.section
|
||||
{
|
||||
margin-top: 50px;
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
document.addEventListener("DOMContentLoaded", function () {
|
||||
var params = window.location.search.substring(1).split("&").reduce(function (params, param) {
|
||||
if (!param) {
|
||||
return params;
|
||||
}
|
||||
|
||||
var values = param.split("=");
|
||||
var name = values[0];
|
||||
var value = values[1];
|
||||
params[name] = value;
|
||||
return params;
|
||||
}, {});
|
||||
|
||||
var form = document.getElementById("feedback-form");
|
||||
for (var name in params) {
|
||||
var input = form.querySelector("[name=" + name + "]");
|
||||
input.value = params[name];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,203 @@
|
||||
NeMo ASR collection API
|
||||
=======================
|
||||
|
||||
|
||||
Model Classes
|
||||
-------------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecCTCModel
|
||||
:show-inheritance:
|
||||
:members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecCTCModelBPE
|
||||
:show-inheritance:
|
||||
:members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecRNNTModel
|
||||
:show-inheritance:
|
||||
:members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecRNNTBPEModel
|
||||
:show-inheritance:
|
||||
:members: transcribe, change_vocabulary, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecClassificationModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
.. autoclass:: nemo.collections.asr.models.EncDecSpeakerLabelModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
|
||||
Modules
|
||||
-------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.ConvASREncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.ConvASRDecoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.ConvASRDecoderClassification
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.SpeakerDecoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.ConformerEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
|
||||
Parts
|
||||
-----
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.submodules.jasper.JasperBlock
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
|
||||
Mixins
|
||||
------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.mixins.mixins.ASRBPEMixin
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.mixins.mixins.ASRModuleMixin
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Datasets
|
||||
--------
|
||||
|
||||
Character Encoding Datasets
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToCharDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.TarredAudioToCharDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToCharWithDursF0Dataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Subword Encoding Datasets
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.AudioToBPEDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.TarredAudioToBPEDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Audio Preprocessors
|
||||
-------------------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.AudioToMFCCPreprocessor
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Audio Augmentors
|
||||
----------------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.modules.CropOrPadSpectrogramAugmentation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.SpeedPerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.TimeStretchPerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.GainPerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.ImpulsePerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.ShiftPerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.NoisePerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.WhiteNoisePerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.RirAndNoisePerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.preprocessing.perturb.TranscodePerturbation
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Miscellaneous Classes
|
||||
---------------------
|
||||
|
||||
RNNT Decoding
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. autoclass:: nemo.collections.asr.metrics.rnnt_wer.RNNTDecoding
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.metrics.rnnt_wer_bpe.RNNTBPEDecoding
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_greedy_decoding.GreedyRNNTInfer
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_greedy_decoding.GreedyBatchedRNNTInfer
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.submodules.rnnt_beam_decoding.BeamRNNTInfer
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
Hypotheses
|
||||
~~~~~~~~~~
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.utils.rnnt_utils.Hypothesis
|
||||
:show-inheritance:
|
||||
:no-members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.utils.rnnt_utils.NBestHypotheses
|
||||
:show-inheritance:
|
||||
:no-members:
|
||||
@@ -0,0 +1,980 @@
|
||||
@article{matchboxnet,
|
||||
title={{MatchboxNet}: 1D Time-Channel Separable Convolutional Neural Network Architecture for Speech Commands Recognition},
|
||||
author={Majumdar, Somshubra and Ginsburg, Boris},
|
||||
journal={Proc. Interspeech 2020},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@article{marblenet,
|
||||
title={MarbleNet: Deep 1D Time-Channel Separable Convolutional Neural Network for Voice Activity Detection},
|
||||
author={Jia, Fei and Majumdar, Somshubra and Ginsburg, Boris},
|
||||
journal={arXiv preprint arXiv:2010.13886},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@inproceedings{panayotov2015librispeech,
|
||||
title={Librispeech: an ASR corpus based on public domain audio books},
|
||||
author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
|
||||
booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2015 IEEE International Conference on},
|
||||
pages={5206--5210},
|
||||
year={2015},
|
||||
organization={IEEE}
|
||||
}
|
||||
|
||||
@article{luong17,
|
||||
author = {Minh{-}Thang Luong and Eugene Brevdo and Rui Zhao},
|
||||
title = {Neural Machine Translation (seq2seq) Tutorial},
|
||||
journal = {https://github.com/tensorflow/nmt},
|
||||
year = {2017},
|
||||
}
|
||||
|
||||
@INPROCEEDINGS{LaurentSeqWiseBN,
|
||||
author={C. {Laurent} and G. {Pereyra} and P. {Brakel} and Y. {Zhang} and Y. {Bengio}},
|
||||
booktitle={2016 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
|
||||
title={Batch normalized recurrent neural networks},
|
||||
year={2016},
|
||||
volume={},
|
||||
number={},
|
||||
pages={2657-2661},
|
||||
keywords={feedforward neural nets;learning (artificial intelligence);recurrent neural nets;speech recognition;batch normalized recurrent neural networks;RNN;sequential data;long-term dependency learning;convergence rate improvement;intermediate representation normalization;feedforward neural networks;speech recognition task;language modeling;training criterion;Training;Recurrent neural networks;Convergence;Speech recognition;Computer architecture;Speech;batch normalization;RNN;LSTM;optimization},
|
||||
doi={10.1109/ICASSP.2016.7472159},
|
||||
ISSN={2379-190X},
|
||||
month={March},}
|
||||
|
||||
@article{graves2005,
|
||||
author = {Alex Graves and Jurgen Schmidhuber},
|
||||
title = {Framewise phoneme classification with bidirectional LSTM and other neural network architectures},
|
||||
journal = {Neural Networks, vol. 18},
|
||||
pages={602–-610},
|
||||
year = {2005},
|
||||
}
|
||||
|
||||
@inproceedings{graves2006,
|
||||
title={Connectionist temporal classification: labelling unsegmented sequence data with recurrent neural networks},
|
||||
author={Graves, Alex and Fern{\'a}ndez, Santiago and Gomez, Faustino and Schmidhuber, J{\"u}rgen},
|
||||
booktitle={Proceedings of the 23rd international conference on Machine learning},
|
||||
pages={369--376},
|
||||
year={2006},
|
||||
organization={ACM}
|
||||
}
|
||||
|
||||
@article{li2019jasper,
|
||||
title={Jasper: An End-to-End Convolutional Neural Acoustic Model},
|
||||
author={Li, Jason and Lavrukhin, Vitaly and Ginsburg, Boris and Leary, Ryan and Kuchaiev, Oleksii and Cohen, Jonathan M and Nguyen, Huyen and Gadde, Ravi Teja},
|
||||
journal={arXiv preprint arXiv:1904.03288},
|
||||
year={2019}
|
||||
}
|
||||
|
||||
@misc{ardila2019common,
|
||||
title={Common Voice: A Massively-Multilingual Speech Corpus},
|
||||
author={Rosana Ardila and Megan Branson and Kelly Davis and Michael Henretty and Michael Kohler and Josh Meyer and Reuben Morais and Lindsay Saunders and Francis M. Tyers and Gregor Weber},
|
||||
year={2019},
|
||||
eprint={1912.06670},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
|
||||
@article{graves2012,
|
||||
title={Sequence Transduction with Recurrent Neural Networks},
|
||||
author={Graves, Alex},
|
||||
journal={arXiv preprint arXiv:1211.3711},
|
||||
year={2012}
|
||||
}
|
||||
|
||||
|
||||
@article{graves2013,
|
||||
title={Generating sequences with recurrent neural networks},
|
||||
author={Graves, Alex},
|
||||
journal={arXiv preprint arXiv:1308.0850},
|
||||
year={2013}
|
||||
}
|
||||
|
||||
@article{sergeev2018horovod,
|
||||
title={Horovod: fast and easy distributed deep learning in TensorFlow},
|
||||
author={Sergeev, Alexander and Del Balso, Mike},
|
||||
journal={arXiv preprint arXiv:1802.05799},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@misc{NVVolta,
|
||||
title = {NVIDIA TESLA V100 GPU ARCHITECTURE},
|
||||
howpublished = {\url{http://images.nvidia.com/content/volta-architecture/pdf/volta-architecture-whitepaper.pdf}},
|
||||
note = {Accessed: 2018-10-09}
|
||||
}
|
||||
|
||||
@article{NVTuring,
|
||||
title = {NVIDIA TURING GPU ARCHITECTURE},
|
||||
howpublished = {\url{https://www.nvidia.com/content/dam/en-zz/Solutions/design-visualization/technologies/turing-architecture/NVIDIA-Turing-Architecture-Whitepaper.pdf}},
|
||||
author = {NVIDIA},
|
||||
year = {2018},
|
||||
note = {Accessed: 2018-10-09}
|
||||
}
|
||||
|
||||
@misc{Rygaard2015,
|
||||
title = {Using Synthesized Speech to Improve Speech Recognition for Low-Resource Languages},
|
||||
author = {Luise Valentin Rygaard},
|
||||
howpublished = {\url{https://parasol.tamu.edu/dreu2015/Rygaard/report.pdf}},
|
||||
year = {2015},
|
||||
}
|
||||
|
||||
@misc{OpenSeq2Seq,
|
||||
title = {OpenSeq2Seq: extensible toolkit for distributed and mixed precision training of sequence-to-sequence models},
|
||||
author = {Kuchaiev, Oleksii and Ginsburg, Boris and Gitman, Igor and Lavrukhin,Vitaly and Case, Carl and Micikevicius, Paulius},
|
||||
howpublished = {\url{https://arxiv.org/abs/1805.10387}},
|
||||
year = {2018},
|
||||
}
|
||||
|
||||
@misc{MPGuide,
|
||||
title = {Training with Mixed Precision},
|
||||
howpublished = {\url{http://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/}},
|
||||
note = {Accessed: 2018-04-06},
|
||||
}
|
||||
|
||||
@misc{Mozilla,
|
||||
title = {Mozilla: A Journey to less than 10\% Word Error Rate},
|
||||
howpublished = {\url{https://hacks.mozilla.org/2017/11/a-journey-to-10-word-error-rate/}},
|
||||
note = {Accessed: 2018-04-06},
|
||||
}
|
||||
|
||||
@article{Waibel1989,
|
||||
title={A time-delay neural network architecture for isolated word recognition},
|
||||
author={Waibel, Alexander, and Hanazawa, Toshiyki and Hinton,Geoffrey and Shirano, Kiyohiro and Lang, Kevin },
|
||||
journal={IEEE Trans. on Acoustics, Speech and Signal Processing},
|
||||
year={1989}
|
||||
}
|
||||
|
||||
@article{Lang1990,
|
||||
title={A time-delay neural network architecture for isolated word recognition},
|
||||
author={Lang, Kevin and Waibel, Alexander, and Hinton,Geoffrey },
|
||||
journal={Neural Networks},
|
||||
year={1990}
|
||||
}
|
||||
|
||||
@book{Bengio1996,
|
||||
Author = {Bengio, Y.},
|
||||
Publisher = {International Thomson Computer Press},
|
||||
Title = {Neural Networks for Speech and Sequence Recognition},
|
||||
Year = {1996}
|
||||
}
|
||||
|
||||
@article{Bengio1992,
|
||||
title={Global optimization of a neural network-hidden Markov model hybrid},
|
||||
author={Bengio, Y., and De Mori, R., and Flammia, G., and Kompe, R. },
|
||||
journal={IEEE Transactions on Neural Networks, 3(2), 252–259},
|
||||
year={1992}
|
||||
}
|
||||
|
||||
@article{Bourlard1994,
|
||||
title={Connectionist speech recognition: a hybrid approach},
|
||||
author={Bourlard, H. A. and Morgan, N.},
|
||||
journal={volume 247 Springer },
|
||||
year={1994}
|
||||
}
|
||||
|
||||
@article{srivastava14a,
|
||||
author = {Nitish Srivastava, and Geoffrey Hinton, and Alex Krizhevsky, and Ilya Sutskever, and Ruslan Salakhutdinov},
|
||||
title = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
|
||||
journal = {Journal of Machine Learning Research},
|
||||
year = {2014},
|
||||
volume = {15},
|
||||
pages = {1929-1958},
|
||||
url = {http://jmlr.org/papers/v15/srivastava14a.html}
|
||||
}
|
||||
|
||||
|
||||
@article{Hinton2012,
|
||||
title={Deep Neural Networks for Acoustic Modeling in Speech Recognition},
|
||||
author={ Hinton,Geoffrey and Deng, Li and Yu, Dong and Dahl,George and Mohamed,Abdel-rahman and Jaitly, Navdeep and Senior, Andrew and Vanhoucke, Vincent and Nguyen, Patrick and Kingsbury, Brian and Sainath, Tara},
|
||||
journal={IEEE Signal Processing Magazine},
|
||||
year={2012}
|
||||
}
|
||||
|
||||
@article{Graves2014,
|
||||
title={Towards End-to-End Speech Recognition with Recurrent Neural Networks},
|
||||
author={Graves, Alex and Jaitly, Navdeep},
|
||||
journal={International Conference on Machine Learning},
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{Chorowski2014,
|
||||
title={End-to-end Continuous Speech Recognition using Attention-based Recurrent NN: First Results},
|
||||
author={ Chorowski, Jan, and Bahdanau, Dzmitry , and Cho, Kyunghyun , and Bengio, Yoshua },
|
||||
journal={Neural Information Processing Systems: Workshop Deep Learning and Representation Learning Workshop },
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{Sak2014,
|
||||
title={Long short-term memory recurrent neural network architectures for large scale acoustic modeling},
|
||||
author={Sak, Hasim and Senior, Andrew and Beaufays, Francoise },
|
||||
journal={Interspeech 2014},
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{Ko2015,
|
||||
title={Audio Augmentation for Speech Recognition},
|
||||
author={Tom, Ko and Vijayaditya, Peddinti and Daniel, Povey
|
||||
and Sanjeev, Khudanpur },
|
||||
journal={Interspeech 2015},
|
||||
year={2015}
|
||||
}
|
||||
|
||||
@article{Tjandra2017,
|
||||
title={Listening while Speaking: Speech Chain by Deep Learning},
|
||||
author={Andros, Tjandra and Sakriani, Sakti and Satoshi, Nakamura },
|
||||
journal={ASRU 2017},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{Tjandra2018,
|
||||
title={Machine Speech Chain with One-shot Speaker Adaptation},
|
||||
author={Andros, Tjandra and Sakriani, Sakti and Satoshi, Nakamura },
|
||||
journal={Interspeech 2018},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{bahdanau2014neural,
|
||||
title={Neural machine translation by jointly learning to align and translate},
|
||||
author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua},
|
||||
journal={arXiv preprint arXiv:1409.0473},
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{cho2014learning,
|
||||
title={Learning phrase representations using RNN encoder-decoder for statistical machine translation},
|
||||
author={Cho, Kyunghyun and Van Merri{\"e}nboer, Bart and Gulcehre, Caglar and Bahdanau, Dzmitry and Bougares, Fethi and Schwenk, Holger and Bengio, Yoshua},
|
||||
journal={arXiv preprint arXiv:1406.1078},
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{rush2015neural,
|
||||
title={A neural attention model for abstractive sentence summarization},
|
||||
author={Rush, Alexander M and Chopra, Sumit and Weston, Jason},
|
||||
journal={arXiv preprint arXiv:1509.00685},
|
||||
year={2015}
|
||||
}
|
||||
|
||||
@article{micikevicius2017mixed,
|
||||
title={Mixed precision training},
|
||||
author={Micikevicius, Paulius and Narang, Sharan and Alben, Jonah and Diamos, Gregory and Elsen, Erich and Garcia, David and Ginsburg, Boris and Houston, Michael and Kuchaev, Oleksii and Venkatesh, Ganesh and others},
|
||||
journal={arXiv preprint arXiv:1710.03740},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@ARTICLE{Britz:2017,
|
||||
author = {{Britz}, Denny and {Goldie}, Anna and {Luong}, Thang and {Le}, Quoc},
|
||||
title = "{Massive Exploration of Neural Machine Translation Architectures}",
|
||||
journal = {ArXiv e-prints arXiv:1703.03906},
|
||||
archivePrefix = "arXiv",
|
||||
eprinttype = {arxiv},
|
||||
eprint = {1703.03906},
|
||||
primaryClass = "cs.CL",
|
||||
keywords = {Computer Science - Computation and Language},
|
||||
year = 2017,
|
||||
month = mar,
|
||||
}
|
||||
|
||||
@inproceedings{vaswani2017attention,
|
||||
title={Attention is all you need},
|
||||
author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
|
||||
booktitle={Advances in Neural Information Processing Systems},
|
||||
pages={6000--6010},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{abadi2016tensorflow,
|
||||
title={TensorFlow: A System for Large-Scale Machine Learning.},
|
||||
author={Abadi, Mart{\'\i}n and Barham, Paul and Chen, Jianmin and Chen, Zhifeng and Davis, Andy and Dean, Jeffrey and Devin, Matthieu and Ghemawat, Sanjay and Irving, Geoffrey and Isard, Michael and others},
|
||||
booktitle={OSDI},
|
||||
volume={16},
|
||||
pages={265--283},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@article{tensor2tensor,
|
||||
author = {Ashish Vaswani and Samy Bengio and Eugene Brevdo and Francois Chollet and Aidan N. Gomez and Stephan Gouws and Llion Jones and \L{}ukasz Kaiser and Nal Kalchbrenner and Niki Parmar and Ryan Sepassi and
|
||||
Noam Shazeer and Jakob Uszkoreit},
|
||||
title = {Tensor2Tensor for Neural Machine Translation},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1803.07416},
|
||||
year = {2018},
|
||||
url = {http://arxiv.org/abs/1803.07416},
|
||||
}
|
||||
|
||||
@article{gehring2017convs2s,
|
||||
author = {Gehring, Jonas, and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N},
|
||||
title = "{Convolutional Sequence to Sequence Learning}",
|
||||
journal = {ArXiv e-prints arXiv:1705.03122},
|
||||
archivePrefix = "arXiv",
|
||||
eprinttype = {arxiv},
|
||||
eprint = {1705.03122},
|
||||
primaryClass = "cs.CL",
|
||||
keywords = {Computer Science - Computation and Language},
|
||||
year = 2017,
|
||||
month = May,
|
||||
}
|
||||
|
||||
@inproceedings{chan2015,
|
||||
title={Listen, attend and spell},
|
||||
author={Chan, William and Jaitly, Navdeep and Le, Quoc V and Vinyals, Oriol},
|
||||
booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2016 IEEE International Conference on},
|
||||
pages={5206--5210},
|
||||
year={2016},
|
||||
organization={IEEE}
|
||||
}
|
||||
|
||||
@inproceedings{xu2015show,
|
||||
title={Show, attend and tell: Neural image caption generation with visual attention},
|
||||
author={Xu, Kelvin and Ba, Jimmy and Kiros, Ryan and Cho, Kyunghyun and Courville, Aaron and Salakhudinov, Ruslan and Zemel, Rich and Bengio, Yoshua},
|
||||
booktitle={International Conference on Machine Learning},
|
||||
pages={2048--2057},
|
||||
year={2015}
|
||||
}
|
||||
|
||||
@incollection{Sutskever2014,
|
||||
title = {Sequence to Sequence Learning with Neural Networks},
|
||||
author = {Sutskever, Ilya and Vinyals, Oriol and Le, Quoc V},
|
||||
booktitle = {Advances in Neural Information Processing Systems 27},
|
||||
editor = {Z. Ghahramani and M. Welling and C. Cortes and N. D. Lawrence and K. Q. Weinberger},
|
||||
pages = {3104--3112},
|
||||
year = {2014},
|
||||
publisher = {Curran Associates, Inc.},
|
||||
url = {http://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks.pdf}
|
||||
}
|
||||
|
||||
@article{DeepSpeech2014,
|
||||
title = {Deep Speech: Scaling up end-to-end speech recognition},
|
||||
author = {Awni Y. Hannun and Carl Case and Jared Casper and Bryan Catanzaro and Greg Diamos and Erich Elsen and Ryan Prenger and Sanjeev Satheesh and Shubho Sengupta and Adam Coates and Andrew Y. Ng},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1412.5567},
|
||||
year = {2014},
|
||||
url = {http://arxiv.org/abs/1412.5567},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1412.5567},
|
||||
timestamp = {Mon, 13 Aug 2018 16:48:07 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/HannunCCCDEPSSCN14},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@inproceedings{DeepSpeech2,
|
||||
author = {Amodei, Dario and Ananthanarayanan, Sundaram and Anubhai, Rishita and Bai, Jingliang and Battenberg, Eric and Case, Carl and Casper, Jared and Catanzaro, Bryan and Cheng, Qiang and Chen, Guoliang and Chen, Jie and Chen, Jingdong and Chen, Zhijie and Chrzanowski, Mike and Coates, Adam and Diamos, Greg and Ding, Ke and Du, Niandong and Elsen, Erich and Engel, Jesse and Fang, Weiwei and Fan, Linxi and Fougner, Christopher and Gao, Liang and Gong, Caixia and Hannun, Awni and Han, Tony and Johannes, Lappi Vaino and Jiang, Bing and Ju, Cai and Jun, Billy and LeGresley, Patrick and Lin, Libby and Liu, Junjie and Liu, Yang and Li, Weigao and Li, Xiangang and Ma, Dongpeng and Narang, Sharan and Ng, Andrew and Ozair, Sherjil and Peng, Yiping and Prenger, Ryan and Qian, Sheng and Quan, Zongfeng and Raiman, Jonathan and Rao, Vinay and Satheesh, Sanjeev and Seetapun, David and Sengupta, Shubho and Srinet, Kavya and Sriram, Anuroop and Tang, Haiyuan and Tang, Liliang and Wang, Chong and Wang, Jidong and Wang, Kaifu and Wang, Yi and Wang, Zhijian and Wang, Zhiqian and Wu, Shuang and Wei, Likai and Xiao, Bo and Xie, Wen and Xie, Yan and Yogatama, Dani and Yuan, Bin and Zhan, Jun and Zhu, Zhenyao},
|
||||
title = {Deep Speech 2: End-to-end Speech Recognition in English and Mandarin},
|
||||
booktitle = {Proceedings of the 33rd International Conference on International Conference on Machine Learning - Volume 48},
|
||||
series = {ICML'16},
|
||||
year = {2016},
|
||||
location = {New York, NY, USA},
|
||||
pages = {173--182},
|
||||
numpages = {10},
|
||||
url = {http://dl.acm.org/citation.cfm?id=3045390.3045410},
|
||||
acmid = {3045410},
|
||||
publisher = {JMLR.org},
|
||||
}
|
||||
|
||||
@inproceedings{prabhavalkar2017comparison,
|
||||
title={A comparison of sequence-to-sequence models for speech recognition},
|
||||
author={Prabhavalkar, Rohit and Rao, Kanishka and Sainath, Tara N and Li, Bo and Johnson, Leif and Jaitly, Navdeep},
|
||||
booktitle={Proc. Interspeech},
|
||||
pages={939--943},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{chiu2017state,
|
||||
title={State-of-the-art speech recognition with sequence-to-sequence models},
|
||||
author={Chiu, Chung-Cheng and Sainath, Tara N and Wu, Yonghui and Prabhavalkar, Rohit and Nguyen, Patrick and Chen, Zhifeng and Kannan, Anjuli and Weiss, Ron J and Rao, Kanishka and Gonina, Katya and others},
|
||||
journal={arXiv preprint arXiv:1712.01769},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@misc{NVMixed,
|
||||
title = {{NVIDA's Mixed-Precision Training - TensorFlow example}},
|
||||
howpublished = {\url{https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/#example_tensorflow}},
|
||||
author={NVIDIA},
|
||||
note = {Accessed: 2018-10-09},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{gehring2017,
|
||||
title={Convolutional sequence to sequence learning},
|
||||
author={Gehring, Jonas and Auli, Michael and Grangier, David and Yarats, Denis and Dauphin, Yann N},
|
||||
journal={arXiv preprint arXiv:1705.03122},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{collobert2016,
|
||||
title={Wav2letter: an end-to-end convnet-based speech recognition system},
|
||||
author={Collobert, Ronan and Puhrsch, Christian and Synnaeve, Gabriel},
|
||||
journal={arXiv preprint arXiv:1609.03193},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@inproceedings{Zhang2016,
|
||||
author={Ying Zhang and Mohammad Pezeshki and Philémon Brakel and Saizheng Zhang and César Laurent and Yoshua Bengio and Aaron Courville},
|
||||
title={Towards End-to-End Speech Recognition with Deep Convolutional Neural Networks},
|
||||
year=2016,
|
||||
booktitle={Interspeech 2016},
|
||||
doi={10.21437/Interspeech.2016-1446},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2016-1446},
|
||||
pages={410--414}
|
||||
}
|
||||
|
||||
@inproceedings{Zhang2017,
|
||||
title={Very deep convolutional networks for end-to-end speech recognition},
|
||||
author={Zhang, Yu, and Chan, William, and Jaitly, Navdeep},
|
||||
booktitle={Acoustics, Speech and Signal Processing (ICASSP), 2017 IEEE International Conference on},
|
||||
year={2017},
|
||||
organization={IEEE}
|
||||
}
|
||||
|
||||
|
||||
@article{Wang2017,
|
||||
title={Tacotron: Towards End-to-End Speech Synthesis},
|
||||
author={ Wang, Yuxuan, and Skerry-Ryan, RJ, and Stanton, Daisy and Wu, Yonghui and Weiss, Ron, and Jaitly, Navdeep and Yang, Zongheng and Xiao, Ying and Chen,Zhifeng and Bengio, Samy and Le, Quoc and Agiomyrgiannakis, Yannis and Clark,Rob and Saurous, Rif A.},
|
||||
journal={arXiv preprint arXiv:1703.10135},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{shen2018natural,
|
||||
title={Natural TTS synthesis by conditioning wavenet on mel spectrogram predictions},
|
||||
author={Shen, Jonathan and Pang, Ruoming and Weiss, Ron J and Schuster, Mike and Jaitly, Navdeep and Yang, Zongheng and Chen, Zhifeng and Zhang, Yu and Wang, Yuxuan and Skerrv-Ryan, Rj and others},
|
||||
booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
|
||||
pages={4779--4783},
|
||||
year={2018},
|
||||
organization={IEEE}
|
||||
}
|
||||
|
||||
@article{griffin1984signal,
|
||||
title={Signal estimation from modified short-time Fourier transform},
|
||||
author={Griffin, Daniel and Lim, Jae},
|
||||
journal={IEEE Transactions on Acoustics, Speech, and Signal Processing},
|
||||
volume={32},
|
||||
number={2},
|
||||
pages={236--243},
|
||||
year={1984},
|
||||
publisher={IEEE}
|
||||
}
|
||||
|
||||
@misc{ito2017lj,
|
||||
title={The LJ speech dataset},
|
||||
author={Ito, Keith and others},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@misc{mailabs,
|
||||
title = {{The M-AILABS Speech Dataset}},
|
||||
howpublished = {\url{http://www.m-ailabs.bayern/en/the-mailabs-speech-dataset/}},
|
||||
author={M-AILABS},
|
||||
note = {Accessed: 2018-10-09},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{merity2016pointer,
|
||||
title={Pointer sentinel mixture models},
|
||||
author={Merity, Stephen and Xiong, Caiming and Bradbury, James and Socher, Richard},
|
||||
journal={arXiv preprint arXiv:1609.07843},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@inproceedings{socher2013recursive,
|
||||
title={Recursive deep models for semantic compositionality over a sentiment treebank},
|
||||
author={Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D and Ng, Andrew and Potts, Christopher},
|
||||
booktitle={Proceedings of the 2013 conference on empirical methods in natural language processing},
|
||||
pages={1631--1642},
|
||||
year={2013}
|
||||
}
|
||||
|
||||
@InProceedings{maas-EtAl:2011:ACL-HLT2011,
|
||||
author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher},
|
||||
title = {Learning Word Vectors for Sentiment Analysis},
|
||||
booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},
|
||||
month = {June},
|
||||
year = {2011},
|
||||
address = {Portland, Oregon, USA},
|
||||
publisher = {Association for Computational Linguistics},
|
||||
pages = {142--150},
|
||||
url = {http://www.aclweb.org/anthology/P11-1015}
|
||||
}
|
||||
|
||||
@inproceedings{Povey2018SemiOrthogonalLM,
|
||||
title={Semi-Orthogonal Low-Rank Matrix Factorization for Deep Neural Networks},
|
||||
author={Daniel Povey and Gaofeng Cheng and Yiming Wang and Ke Li and Hainan Xu and Mahsa Yarmohammadi and Sanjeev Khudanpur},
|
||||
booktitle={Interspeech},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{CAPIO2017,
|
||||
author = {Kyu J. Han and Akshay Chandrashekaran and Jungsuk Kim and Ian R. Lane},
|
||||
title = {The {CAPIO} 2017 Conversational Speech Recognition System},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1801.00059},
|
||||
year = {2018},
|
||||
url = {http://arxiv.org/abs/1801.00059},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1801.00059},
|
||||
timestamp = {Mon, 13 Aug 2018 16:49:10 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1801-00059},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{WaveNet,
|
||||
author = {A{\"{a}}ron van den Oord and Sander Dieleman and Heiga Zen and Karen Simonyan and Oriol Vinyals and Alex Graves and Nal Kalchbrenner and Andrew W. Senior and Koray Kavukcuoglu},
|
||||
title = {WaveNet: {A} Generative Model for Raw Audio},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1609.03499},
|
||||
year = {2016},
|
||||
url = {http://arxiv.org/abs/1609.03499},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1609.03499},
|
||||
timestamp = {Mon, 13 Aug 2018 16:49:15 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/OordDZSVGKSK16},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{FacebookGERENGBackTranslation,
|
||||
author = {Rico Sennrich and Barry Haddow and Alexandra Birch},
|
||||
title = {Improving Neural Machine Translation Models with Monolingual Data},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1511.06709},
|
||||
year = {2015},
|
||||
url = {http://arxiv.org/abs/1511.06709},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1511.06709},
|
||||
timestamp = {Mon, 13 Aug 2018 16:47:05 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/SennrichHB15a},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{GlobalStyleTokens,
|
||||
author = {Yuxuan Wang and Daisy Stanton and Yu Zhang and R. J. Skerry{-}Ryan and Eric Battenberg and Joel Shor and Ying Xiao and Fei Ren and Ye Jia and Rif A. Saurous},
|
||||
title = {Style Tokens: Unsupervised Style Modeling, Control and Transfer in End-to-End Speech Synthesis},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1803.09017},
|
||||
year = {2018},
|
||||
url = {http://arxiv.org/abs/1803.09017},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1803.09017},
|
||||
timestamp = {Mon, 13 Aug 2018 16:46:53 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1803-09017},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{IoffeS15BatchNorm,
|
||||
author = {Sergey Ioffe and Christian Szegedy},
|
||||
title = {Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1502.03167},
|
||||
year = {2015},
|
||||
url = {http://arxiv.org/abs/1502.03167},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1502.03167},
|
||||
timestamp = {Mon, 13 Aug 2018 16:47:06 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/IoffeS15},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{kingma,
|
||||
author = {Diederik P. Kingma and
|
||||
Jimmy Ba},
|
||||
title = {Adam: {A} Method for Stochastic Optimization},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1412.6980},
|
||||
year = {2014},
|
||||
url = {http://arxiv.org/abs/1412.6980},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1412.6980},
|
||||
timestamp = {Mon, 13 Aug 2018 01:00:00 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/KingmaB14},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@incollection{Salimans2016WeightNorm,
|
||||
title = {Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks},
|
||||
author = {Salimans, Tim and Kingma, Durk P},
|
||||
booktitle = {Advances in Neural Information Processing Systems 29},
|
||||
editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett},
|
||||
pages = {901--909},
|
||||
year = {2016},
|
||||
publisher = {Curran Associates, Inc.},
|
||||
url = {http://papers.nips.cc/paper/6114-weight-normalization-a-simple-reparameterization-to-accelerate-training-of-deep-neural-networks.pdf}
|
||||
}
|
||||
|
||||
@article{wu2016google,
|
||||
title={Google's neural machine translation system: Bridging the gap between human and machine translation},
|
||||
author={Wu, Yonghui and Schuster, Mike and Chen, Zhifeng and Le, Quoc V and Norouzi, Mohammad and Macherey, Zolfgang and Krikun, Maxim and Cao, Yuan and Gao, Qin and Macherey, Klaus and others},
|
||||
journal={arXiv preprint arXiv:1609.08144},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@inproceedings{opennmt,
|
||||
author = {Guillaume Klein and Yoon Kim and Yuntian Deng and Jean Senellart and Alexander M. Rush},
|
||||
title = {OpenNMT: Open-Source Toolkit for Neural Machine Translation},
|
||||
booktitle = {Proc. ACL},
|
||||
year = {2017},
|
||||
url = {https://doi.org/10.18653/v1/P17-4012},
|
||||
doi = {10.18653/v1/P17-4012}
|
||||
}
|
||||
|
||||
@article{paszke2017automatic,
|
||||
title={Automatic differentiation in PyTorch},
|
||||
author={Paszke, Adam and Gross, Sam and Chintala, Soumith and Chanan, Gregory and Yang, Edward and DeVito, Zachary and Lin, Zeming and Desmaison, Alban and Antiga, Luca and Lerer, Adam},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{yu2014introduction,
|
||||
title={An introduction to computational networks and the computational network toolkit},
|
||||
author={Yu, Dong and Eversole, Adam and Seltzer, Mike and Yao, Kaisheng and Huang, Zhiheng and Guenter, Brian and Kuchaiev, Oleksii and Zhang, Yu and Seide, Frank and Wang, Huaming and others},
|
||||
journal={Microsoft Technical Report MSR-TR-2014--112},
|
||||
year={2014}
|
||||
}
|
||||
|
||||
@article{nvidia2017v100,
|
||||
title={V100 GPU architecture. The world’s most advanced data center GPU. Version WP-08608-001\_v1. 1},
|
||||
author={NVIDIA, Tesla},
|
||||
journal={NVIDIA. Aug},
|
||||
pages={108},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{post2018call,
|
||||
title={A call for clarity in reporting bleu scores},
|
||||
author={Post, Matt},
|
||||
journal={arXiv preprint arXiv:1804.08771},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{Ba2016LayerNorm,
|
||||
author = {Jimmy Lei Ba and Jamie Ryan Kiros and Geoffrey E Hinton},
|
||||
title = {Layer normalization},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1607.06450},
|
||||
year = {2016},
|
||||
url = {http://arxiv.org/abs/1607.06450},
|
||||
archivePrefix = {arXiv},
|
||||
}
|
||||
|
||||
@inproceedings{Dauphin2017GLU,
|
||||
author = {Dauphin, Yann N. and Fan, Angela and Auli, Michael and Grangier, David},
|
||||
title = {Language Modeling with Gated Convolutional Networks},
|
||||
booktitle = {Proceedings of the 34th International Conference on Machine Learning - Volume 70},
|
||||
series = {ICML'17},
|
||||
year = {2017},
|
||||
location = {Sydney, NSW, Australia},
|
||||
pages = {933--941},
|
||||
numpages = {9},
|
||||
url = {http://dl.acm.org/citation.cfm?id=3305381.3305478},
|
||||
acmid = {3305478},
|
||||
publisher = {JMLR.org},
|
||||
}
|
||||
|
||||
@incollection{Oord2016PixelCNN,
|
||||
title = {Conditional Image Generation with PixelCNN Decoders},
|
||||
author = {van den Oord, Aaron and Kalchbrenner, Nal and Espeholt, Lasse and kavukcuoglu, koray and Vinyals, Oriol and Graves, Alex},
|
||||
booktitle = {Advances in Neural Information Processing Systems 29},
|
||||
editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett},
|
||||
pages = {4790--4798},
|
||||
year = {2016},
|
||||
publisher = {Curran Associates, Inc.},
|
||||
url = {http://papers.nips.cc/paper/6527-conditional-image-generation-with-pixelcnn-decoders.pdf}
|
||||
}
|
||||
|
||||
@article{he2015,
|
||||
title={Deep residual learning for image recognition},
|
||||
author={K. He, and X. Zhang, and S. Ren, and J. Sun},
|
||||
journal={arXiv preprint arXiv:1512.03385},
|
||||
year={2015}
|
||||
}
|
||||
|
||||
@article{huang2016,
|
||||
title={Densely Connected Convolutional Networks},
|
||||
author={Gao Huang, and Zhuang Liu, and Laurens van der Maaten, and Kilian Q. Weinberger},
|
||||
journal={arXiv preprint arXiv:1608.06993},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@inproceedings{heafield2011kenlm,
|
||||
title={KenLM: Faster and smaller language model queries},
|
||||
author={Heafield, Kenneth},
|
||||
booktitle={Proceedings of the sixth workshop on statistical machine translation},
|
||||
pages={187--197},
|
||||
year={2011},
|
||||
organization={Association for Computational Linguistics}
|
||||
}
|
||||
|
||||
@article{dai2018transformer,
|
||||
title={Transformer-XL: Language Modeling with Longer-Term Dependency},
|
||||
author={Dai, Zihang and Yang, Zhilin and Yang, Yiming and Cohen, William W and Carbonell, Jaime and Le, Quoc V and Salakhutdinov, Ruslan},
|
||||
year={2018},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1901.02860},
|
||||
url = {http://arxiv.org/abs/1901.02860},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1901.02860},
|
||||
timestamp = {Fri, 01 Feb 2019 13:39:59 +0100},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1901-02860},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@inproceedings{Saon+2016,
|
||||
author={George Saon and Tom Sercu and Steven Rennie and Hong-Kwang J. Kuo},
|
||||
title={The IBM 2016 English Conversational Telephone Speech Recognition System},
|
||||
year=2016,
|
||||
booktitle={Interspeech 2016},
|
||||
doi={10.21437/Interspeech.2016-1460},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2016-1460},
|
||||
pages={7--11}
|
||||
}
|
||||
|
||||
@INPROCEEDINGS{Sercu-2016,
|
||||
author={T. {Sercu} and C. {Puhrsch} and B. {Kingsbury} and Y. {LeCun}},
|
||||
booktitle={2016 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
|
||||
title={Very deep multilingual convolutional neural networks for LVCSR},
|
||||
year={2016},
|
||||
volume={},
|
||||
number={},
|
||||
pages={4955-4959},
|
||||
keywords={natural language processing;neural nets;speech recognition;very deep multilingual convolutional neural networks;LVCSR;CNN;large vocabulary continuous speech recognition systems;word error rate;Training;Context;Hidden Markov models;Neural networks;Computer architecture;Kernel;Training data;Convolutional Networks;Multilingual;Acoustic Modeling;Speech Recognition;Neural Networks},
|
||||
doi={10.1109/ICASSP.2016.7472620},
|
||||
ISSN={2379-190X},
|
||||
month={March},}
|
||||
|
||||
|
||||
@inproceedings{Sercu+2016,
|
||||
author={Tom Sercu and Vaibhava Goel},
|
||||
title={Advances in Very Deep Convolutional Neural Networks for LVCSR},
|
||||
year=2016,
|
||||
booktitle={Interspeech 2016},
|
||||
doi={10.21437/Interspeech.2016-1033},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2016-1033},
|
||||
pages={3429--3433}
|
||||
}
|
||||
|
||||
@INPROCEEDINGS{Xiong-2018,
|
||||
author={W. {Xiong} and L. {Wu} and F. {Alleva} and J. {Droppo} and X. {Huang} and A. {Stolcke}},
|
||||
booktitle={2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
|
||||
title={The Microsoft 2017 Conversational Speech Recognition System},
|
||||
year={2018},
|
||||
volume={},
|
||||
number={},
|
||||
pages={5934-5938},
|
||||
keywords={convolution;feedforward neural nets;natural language processing;speaker recognition;speech processing;language model rescoring step;senone level;switchboard domains;character-based LSTM language models;NIST 2000 switchboard test set;frame level;word-level voting;acoustic model posteriors;dialog session aware LSTM language models;CNN-BLSTM acoustic model;Microsoft 2017 conversational speech recognition system;Acoustics;Error analysis;Training;Speech recognition;Switches;Computational modeling;Context modeling;Conversational speech recognition;CNN;LACE;BLSTM;LSTM-LM;system combination;human parity},
|
||||
doi={10.1109/ICASSP.2018.8461870},
|
||||
ISSN={2379-190X},
|
||||
month={April},}
|
||||
|
||||
@inproceedings{zeyer2018improved,
|
||||
author={Albert Zeyer and Kazuki Irie and Ralf Schlüter and Hermann Ney},
|
||||
title={Improved Training of End-to-end Attention Models for Speech Recognition},
|
||||
year=2018,
|
||||
booktitle={Proc. Interspeech 2018},
|
||||
pages={7--11},
|
||||
doi={10.21437/Interspeech.2018-1616},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2018-1616}
|
||||
}
|
||||
|
||||
@article{Wav2LetterV2,
|
||||
author = {Vitaliy Liptchinsky and
|
||||
Gabriel Synnaeve and
|
||||
Ronan Collobert},
|
||||
title = {Letter-Based Speech Recognition with Gated ConvNets},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1712.09444},
|
||||
year = {2017},
|
||||
url = {http://arxiv.org/abs/1712.09444},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1712.09444},
|
||||
timestamp = {Mon, 13 Aug 2018 16:46:33 +0200},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1712-09444},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{zeghidour2018,
|
||||
author = {Neil Zeghidour and
|
||||
Qiantong Xu and
|
||||
Vitaliy Liptchinsky and
|
||||
Nicolas Usunier and
|
||||
Gabriel Synnaeve and
|
||||
Ronan Collobert},
|
||||
title = {Fully Convolutional Speech Recognition},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1812.06864},
|
||||
year = {2018},
|
||||
url = {http://arxiv.org/abs/1812.06864},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1812.06864},
|
||||
timestamp = {Tue, 01 Jan 2019 15:01:25 +0100},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1812-06864},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@inproceedings{Hadian2018,
|
||||
author={Hossein Hadian and Hossein Sameti and Daniel Povey and Sanjeev Khudanpur},
|
||||
title={End-to-end Speech Recognition Using Lattice-free MMI},
|
||||
year=2018,
|
||||
booktitle={Proc. Interspeech 2018},
|
||||
pages={12--16},
|
||||
doi={10.21437/Interspeech.2018-1423},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2018-1423}
|
||||
}
|
||||
|
||||
@inproceedings{Tang2018,
|
||||
author={Jian Tang and Yan Song and Lirong Dai and Ian McLoughlin},
|
||||
title={Acoustic Modeling with Densely Connected Residual Network for Multichannel Speech Recognition},
|
||||
year=2018,
|
||||
booktitle={Proc. Interspeech 2018},
|
||||
pages={1783--1787},
|
||||
doi={10.21437/Interspeech.2018-1089},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2018-1089}
|
||||
}
|
||||
|
||||
@article{Kurata2017LanguageMW,
|
||||
title={Language modeling with highway LSTM},
|
||||
author={Gakuto Kurata and Bhuvana Ramabhadran and George Saon and Abhinav Sethy},
|
||||
journal={2017 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)},
|
||||
year={2017},
|
||||
pages={244-251}
|
||||
}
|
||||
|
||||
@inproceedings{Saon2017,
|
||||
author={George Saon and Gakuto Kurata and Tom Sercu and Kartik Audhkhasi and Samuel Thomas and Dimitrios Dimitriadis and Xiaodong Cui and Bhuvana Ramabhadran and Michael Picheny and Lynn-Li Lim and Bergul Roomi and Phil Hall},
|
||||
title={English Conversational Telephone Speech Recognition by Humans and Machines},
|
||||
year=2017,
|
||||
booktitle={Proc. Interspeech 2017},
|
||||
pages={132--136},
|
||||
doi={10.21437/Interspeech.2017-405},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2017-405}
|
||||
}
|
||||
|
||||
@inproceedings{Povey+2016,
|
||||
author={Daniel Povey and Vijayaditya Peddinti and Daniel Galvez and Pegah Ghahremani and Vimal Manohar and Xingyu Na and Yiming Wang and Sanjeev Khudanpur},
|
||||
title={Purely Sequence-Trained Neural Networks for ASR Based on Lattice-Free MMI},
|
||||
year=2016,
|
||||
booktitle={Interspeech 2016},
|
||||
doi={10.21437/Interspeech.2016-595},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2016-595},
|
||||
pages={2751--2755}
|
||||
}
|
||||
|
||||
@article{Yang2018,
|
||||
author = {Xuerui Yang and
|
||||
Jiwei Li and
|
||||
Xi Zhou},
|
||||
title = {A novel pyramidal-FSMN architecture with lattice-free {MMI} for speech
|
||||
recognition},
|
||||
journal = {CoRR},
|
||||
volume = {abs/1810.11352},
|
||||
year = {2018},
|
||||
url = {http://arxiv.org/abs/1810.11352},
|
||||
archivePrefix = {arXiv},
|
||||
eprint = {1810.11352},
|
||||
timestamp = {Wed, 31 Oct 2018 14:24:29 +0100},
|
||||
biburl = {https://dblp.org/rec/bib/journals/corr/abs-1810-11352},
|
||||
bibsource = {dblp computer science bibliography, https://dblp.org}
|
||||
}
|
||||
|
||||
@article{liptchinsky2017based,
|
||||
title={Letter-Based Speech Recognition with Gated ConvNets},
|
||||
author={Liptchinsky, Vitaliy and Synnaeve, Gabriel and Collobert, Ronan},
|
||||
journal={arXiv preprint arXiv:1712.09444},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@inproceedings{Weng2018,
|
||||
author={Chao Weng and Jia Cui and Guangsen Wang and Jun Wang and Chengzhu Yu and Dan Su and Dong Yu},
|
||||
title={Improving Attention Based Sequence-to-Sequence Models for End-to-End English Conversational Speech Recognition},
|
||||
year=2018,
|
||||
booktitle={Proc. Interspeech 2018},
|
||||
pages={761--765},
|
||||
doi={10.21437/Interspeech.2018-1030},
|
||||
url={http://dx.doi.org/10.21437/Interspeech.2018-1030}
|
||||
}
|
||||
|
||||
@INPROCEEDINGS{Battenberg2017,
|
||||
author={E. {Battenberg} and J. {Chen} and R. {Child} and A. {Coates} and Y. G. Y. {Li} and H. {Liu} and S. {Satheesh} and A. {Sriram} and Z. {Zhu}},
|
||||
booktitle={2017 IEEE Automatic Speech Recognition and Understanding Workshop (ASRU)},
|
||||
title={Exploring neural transducers for end-to-end speech recognition},
|
||||
year={2017},
|
||||
volume={},
|
||||
number={},
|
||||
pages={206-213},
|
||||
keywords={recurrent neural nets;speech recognition;Hub500 benchmark;CTC models;speech recognition pipeline;RNN-Transducer models;language model;Seq2Seq models;end-to-end speech recognition;neural transducers;Decoding;Hidden Markov models;Transducers;Task analysis;Speech;Mathematical model;Neural networks},
|
||||
doi={10.1109/ASRU.2017.8268937},
|
||||
ISSN={},
|
||||
month={Dec},
|
||||
}
|
||||
|
||||
@inproceedings{
|
||||
loshchilov2018,
|
||||
title={Decoupled Weight Decay Regularization},
|
||||
author={Ilya Loshchilov and Frank Hutter},
|
||||
booktitle={International Conference on Learning Representations},
|
||||
year={2019},
|
||||
url={https://openreview.net/forum?id=Bkg6RiCqY7},
|
||||
}
|
||||
|
||||
@article{zhang2017ndadam,
|
||||
author = {Zijun Zhang and Lin Ma and Zongpeng Li and Chuan Wu},
|
||||
title = {Normalized Direction-preserving Adam},
|
||||
journal = {arXiv e-prints arXiv:1709.04546},
|
||||
year = {2017},
|
||||
}
|
||||
|
||||
@article{park2019,
|
||||
author = {{Park}, Daniel S. and {Chan}, William and {Zhang}, Yu and
|
||||
{Chiu}, Chung-Cheng and {Zoph}, Barret and {Cubuk}, Ekin D. and
|
||||
{Le}, Quoc V.},
|
||||
title = "{SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition}",
|
||||
journal = {arXiv e-prints},
|
||||
year = "2019",
|
||||
eid = {arXiv:1904.08779},
|
||||
eprint = {1904.08779},
|
||||
}
|
||||
|
||||
@article{novograd2019,
|
||||
author = {{Ginsburg}, Boris and {Castonguay}, Patrice and {Hrinchuk}, Oleksii and
|
||||
{Kuchaiev}, Oleksii and {Lavrukhin}, Vitaly and {Leary}, Ryan and
|
||||
{Li}, Jason and {Nguyen}, Huyen and {Cohen}, Jonathan M.},
|
||||
title = "{Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks}",
|
||||
journal = {arXiv e-prints},
|
||||
year = "2019",
|
||||
eid = {arXiv:1905.11286},
|
||||
eprint = {1905.11286},
|
||||
}
|
||||
|
||||
@article{kriman2019quartznet,
|
||||
title={Quartznet: {Deep} automatic speech recognition with 1d time-channel separable convolutions},
|
||||
author={Kriman, Samuel and Beliaev, Stanislav and Ginsburg, Boris and Huang, Jocelyn and Kuchaiev, Oleksii and Lavrukhin, Vitaly and Leary, Ryan and Li, Jason and Zhang, Yang},
|
||||
journal={arXiv preprint arXiv:1910.10261},
|
||||
year={2019}
|
||||
}
|
||||
|
||||
@misc{itu1988g711,
|
||||
title={{ITU-T} {G.711} - {Pulse} code modulation ({PCM}) of voice frequencies},
|
||||
author={ITU-T Geneva Switzerland},
|
||||
year={1988},
|
||||
}
|
||||
|
||||
@article{han2020contextnet,
|
||||
title={ContextNet: Improving convolutional neural networks for automatic speech recognition with global context},
|
||||
author={Han, Wei and Zhang, Zhengdong and Zhang, Yu and Yu, Jiahui and Chiu, Chung-Cheng and Qin, James and Gulati, Anmol and Pang, Ruoming and Wu, Yonghui},
|
||||
journal={arXiv:2005.03191},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@inproceedings{hu2018squeeze,
|
||||
title={Squeeze-and-excitation networks},
|
||||
author={Hu, Jie and Shen, Li and Sun, Gang},
|
||||
booktitle={ICVPR},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{koluguri2020speakernet,
|
||||
title={SpeakerNet: 1D Depth-wise Separable Convolutional Network for Text-Independent Speaker Recognition and Verification},
|
||||
author={Koluguri, Nithin Rao and Li, Jason and Lavrukhin, Vitaly and Ginsburg, Boris},
|
||||
journal={arXiv preprint arXiv:2010.12653},
|
||||
year={2020}
|
||||
}
|
||||
|
||||
@article{gulati2020conformer,
|
||||
title={Conformer: Convolution-augmented transformer for speech recognition},
|
||||
author={Gulati, Anmol and Qin, James and Chiu, Chung-Cheng and Parmar, Niki and Zhang, Yu and Yu, Jiahui and Han, Wei and Wang, Shibo and Zhang, Zhengdong and Wu, Yonghui and others},
|
||||
journal={arXiv preprint arXiv:2005.08100},
|
||||
year={2020}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
#####################
|
||||
ASR Language Modeling
|
||||
#####################
|
||||
|
||||
Language models have shown to help the accuracy of ASR models. NeMo supports the following two approaches to incorporate language models into the ASR models:
|
||||
|
||||
* :ref:`ngram_modeling`
|
||||
* :ref:`neural_rescoring`
|
||||
|
||||
It is possible to use both approaches on the same ASR model.
|
||||
|
||||
|
||||
.. _ngram_modeling:
|
||||
|
||||
************************
|
||||
N-gram Language Modeling
|
||||
************************
|
||||
|
||||
In this approach, an N-gram LM is trained on text data, then it is used in fusion with beam search decoding to find the
|
||||
best candidates. The beam search decoders in NeMo support language models trained with KenLM library (
|
||||
`https://github.com/kpu/kenlm <https://github.com/kpu/kenlm>`__).
|
||||
The beam search decoders and KenLM library are not installed by default in NeMo, and you need to install them to be
|
||||
able to use beam search decoding and N-gram LM.
|
||||
Please refer to `scripts/ngram_lm/install_beamsearch_decoders.sh` on how to install them.
|
||||
|
||||
NeMo supports both character-based and BPE-based models for N-gram LMs. An N-gram LM can be used with beam search
|
||||
decoders on top of the ASR models to produce more accurate candidates. The beam search decoder would incorporate
|
||||
the scores produced by the N-gram LM into its score calculations as the following:
|
||||
|
||||
.. code::
|
||||
|
||||
final_score = acoustic_score + beam_alpha*lm_score + beam_beta*seq_length
|
||||
|
||||
where acoustic_score is the score predicted by the acoustic encoder and lm_score is the one estimated by the LM.
|
||||
Parameter 'beam_alpha' specifies amount of importance to place on the N-gram language model, and 'beam_beta' is a
|
||||
penalty term to consider the sequence length in the scores. Larger alpha means more importance on the LM and less
|
||||
importance on the acoustic model. Negative values for beta will give penalty to longer sequences and make the decoder
|
||||
to prefer shorter predictions, while positive values would result in longer candidates.
|
||||
|
||||
|
||||
Train N-gram LM
|
||||
===============
|
||||
|
||||
The script to train an N-gram language model with KenLM can be found at
|
||||
`scripts/asr_language_modeling/ngram_lm/train_kenlm.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/ngram_lm/train_kenlm.py>`__.
|
||||
|
||||
This script would train an N-gram language model with KenLM library which can be used with the beam search decoders
|
||||
on top of the ASR models. This script supports both character level and BPE level encodings and models which is
|
||||
detected automatically from the type of the model.
|
||||
|
||||
|
||||
You may train the N-gram model as the following:
|
||||
|
||||
.. code::
|
||||
|
||||
python train_kenlm.py --nemo_model_file <path to the .nemo file of the model> \
|
||||
--train_file <path to the training text or JSON manifest file \
|
||||
--kenlm_bin_path <path to the bin folder of KenLM library> \
|
||||
--kenlm_model_file <path to store the binary KenLM model> \
|
||||
--ngram_length <order of N-gram model>
|
||||
|
||||
The train file specified by `--train_file` can be a text file or JSON manifest. If the file's extension is anything
|
||||
other than `.json`, it assumes that data format is plain text. For plain text format, each line should contain one
|
||||
sample. For JSON manifest file, the file need to contain json formatted samples per each line like this:
|
||||
|
||||
.. code::
|
||||
|
||||
{"audio_filepath": "/data_path/file1.wav", "text": "The transcript of the audio file."}
|
||||
|
||||
It just extracts the `text` field from each line to create the training text file. After the N-gram model is trained,
|
||||
it is stored at the path specified by `--kenlm_model_file`.
|
||||
|
||||
The following is the list of the arguments for the training script:
|
||||
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| **Argument** | **Type** | **Default** | **Description** |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| nemo_model_file | str | Required | The path of the `.nemo` file of the ASR model. It is needed to extract the tokenizer. |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| train_file | str | Required | Path to the training file, it can be a text file or JSON manifest. |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| kenlm_model_file | str | Required | The path to store the KenLM binary model file. |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| kenlm_bin_path | str | Required | The path to the bin folder of KenLM. It is a folder named `bin` under where KenLM is installed.|
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| ngram_length** | int | Required | Specifies order of N-gram LM. |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
| do_lower_case | bool | ``False`` | Whether to make the training text all lower case. |
|
||||
+------------------+----------+-------------+------------------------------------------------------------------------------------------------+
|
||||
|
||||
** Note: Recommend to use 6 as the order of the N-gram model for BPE-based models. Higher orders may need the re-compilation of KenLM to support it.
|
||||
|
||||
Evaluate by Beam Search Decoding and N-gram LM
|
||||
==============================================
|
||||
|
||||
NeMo's beam search decoders are capable of using the KenLM's N-gram models to find the best candidates.
|
||||
The script to evaluate an ASR model with beam search decoding and N-gram models can be found at
|
||||
`scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py>`__.
|
||||
|
||||
You may evaluate an ASR model as the following:
|
||||
|
||||
.. code::
|
||||
|
||||
python eval_beamsearch_ngram.py --nemo_model_file <path to the .nemo file of the model> \
|
||||
--input_manifest <path to the evaluation JSON manifest file \
|
||||
--kenlm_model_file <path to the binary KenLM model> \
|
||||
--acoustic_batch_size <batch size for calculating log probabilities> \
|
||||
--beam_width <list of the beam widths> \
|
||||
--beam_alpha <list of the beam alphas> \
|
||||
--beam_width <list of the beam betas> \
|
||||
--preds_output_folder <optional folder to store the predictions> \
|
||||
--decoding_mode beam_search_ngram
|
||||
|
||||
It can evaluate a model in the three following modes by setting the argument `--decoding_mode`:
|
||||
|
||||
* greedy: Just greedy decoding is done, and no beam search decoding is performed.
|
||||
* beamsearch: The beam search decoding is done but without using the N-gram language model, final results would be equivalent to setting the weight of LM (beam_beta) to zero.
|
||||
* beamsearch_ngram: The beam search decoding is done with N-gram LM.
|
||||
|
||||
The `beamsearch` mode would evaluate by beam search decoding without any language model.
|
||||
It would report the performances in terms of Word Error Rate (WER) and Character Error Rate (CER). Moreover,
|
||||
the WER/CER of the model when the best candidate is selected among the candidates is also reported as the best WER/CER.
|
||||
It can be an indicator of how good the predicted candidates are.
|
||||
|
||||
The script would initially load the ASR model and predict the outputs of the model's encoder as log probabilities.
|
||||
This part would be computed in batches on a device selected by `--device`, which can be CPU (`--device=cpu`) or a
|
||||
single GPU (`--device=cuda:0`). The batch size of this part can get specified by `--acoustic_batch_size`. You may use
|
||||
the largest batch size feasible to speed up the step of calculating the log probabilities. You may also use `--use_amp`
|
||||
to speed up the calculation of log probabilities and make it possible to use larger sizes for `--acoustic_batch_size`.
|
||||
Currently multi-GPU is not supported for calculating the log probabilities, but using `--probs_cache_file` can help.
|
||||
It stores the log probabilities produced from the model's encoder into a pickle file so that next time the first step
|
||||
can get skipped.
|
||||
|
||||
The following is the list of the arguments for the evaluation script:
|
||||
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| **Argument** |**Type**| **Default** | **Description** |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| nemo_model_file | str | Required | The path of the `.nemo` file of the ASR model to extract the tokenizer. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| input_manifest | str | Required | Path to the training file, it can be a text file or JSON manifest. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| kenlm_model_file | str | Required | The path to store the KenLM binary model file. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| preds_output_folder | str | None | The path to an optional folder to store the predictions. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| probs_cache_file | str | None | The cache file for storing the outputs of the model. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| acoustic_batch_size | int | 16 | The batch size to calculate log probabilities. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| use_amp | bool | ``False`` | Whether to use AMP if available to calculate log probabilities. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| device | str | cuda | The device to load the model onto to calculate log probabilities. |
|
||||
| | | | It can `cpu`, `cuda`, `cuda:0`, `cuda:1`, ... |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| decoding_mode | str | beamsearch_ngram | The decoding scheme to be used for evaluation. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beam_width | float | Required | The width or list of the widths of the beam search decoding. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beam_alpha | float | Required | The alpha parameter or list of the alphas for the beam search decoding. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beam_beta | float | Required | The beta parameter or list of the betas for the beam search decoding. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beam_batch_size | int | 128 | The batch size to be used for beam search decoding. |
|
||||
| | | | Larger batch size can be a little faster, but uses larger memory. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
|
||||
Width of the beam search (`--beam_width`) specifies the number of top candidates/predictions the beam search decoder
|
||||
would search for. Larger beams result in more accurate but slower predictions.
|
||||
|
||||
There is also a tutorial to learn more about evaluating the ASR models with N-gram LM here:
|
||||
`Offline ASR Inference with Beam Search and External Language Model Rescoring <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/Offline_ASR.ipynb>`_
|
||||
|
||||
Hyperparameter Grid Search
|
||||
--------------------------
|
||||
|
||||
Beam search decoding with N-gram LM has three main hyperparameters: `beam_width`, `beam_alpha`, and `beam_beta`.
|
||||
The accuracy of the model is dependent to the values of these parameters, specially beam_alpha and beam_beta.
|
||||
You may specify a single or list of values for each of these parameters to perform grid search. It would perform the
|
||||
beam search decoding on all the combinations of the these three hyperparameters.
|
||||
For instance, the following set of parameters would results in 2*1*2=4 beam search decodings:
|
||||
|
||||
.. code::
|
||||
|
||||
python eval_beamsearch_ngram.py ... \
|
||||
--beam_width 64 128 \
|
||||
--beam_alpha 1.0 \
|
||||
--beam_beta 1.0 0.5
|
||||
|
||||
|
||||
.. _neural_rescoring:
|
||||
|
||||
****************
|
||||
Neural Rescoring
|
||||
****************
|
||||
|
||||
In this approach a neural network is used which can gives scores to a candidate. A candidate is the text transcript predicted by the decoder of the ASR model.
|
||||
The top K candidates produced by the beam search decoding (beam width of K) are given to a neural language model to rank them.
|
||||
Ranking can be done by a language model which gives a score to each candidate.
|
||||
This score is usually combined with the scores from the beam search decoding to produce the final scores and rankings.
|
||||
|
||||
Train Neural Rescorer
|
||||
=====================
|
||||
|
||||
An example script to train such a language model with Transformer can be found at `examples/nlp/language_modeling/transformer_lm.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/language_modeling/transformer_lm.py>`__.
|
||||
It trains a TransformerLMModel which can be used as a neural rescorer for an ASR system.
|
||||
|
||||
|
||||
Evaluation
|
||||
==========
|
||||
|
||||
Given a trained TransformerLMModel `.nemo` file, the script available at
|
||||
`scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py>`__
|
||||
can be used to re-score beams obtained with ASR model. You need the `.tsv` file containing the candidates produced
|
||||
by the acoustic model and the beam search decoding to use this script. The candidates can be the result of just the beam
|
||||
search decoding or the result of fusion with an N-gram LM. You may generate this file by specifying `--preds_output_folder' for
|
||||
`scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py>`__.
|
||||
|
||||
The neural rescorer would rescore the beams/candidates by using two parameters of `rescorer_alpha` and `rescorer_beta` as the following:
|
||||
|
||||
.. code::
|
||||
|
||||
final_score = beam_search_score + rescorer_alpha*neural_rescorer_score + rescorer_beta*seq_length
|
||||
|
||||
Parameter `rescorer_alpha` specifies amount of importance to place on the neural rescorer model, and `rescorer_beta` is
|
||||
a penalty term to consider the sequence length in the scores. They have similar effects like the parameters
|
||||
`beam_alpha` and `beam_beta` of beam search decoder and N-gram LM.
|
||||
|
||||
You may follow the following steps to evaluate a neural LM:
|
||||
|
||||
#. Obtain `.tsv` file with beams and their corresponding scores. Scores can be from a regular beam search decoder or
|
||||
in fusion with an N-gram LM scores. For a given beam size `beam_size` and a number of examples
|
||||
for evaluation `num_eval_examples`, it should contain (`num_eval_examples` x `beam_size`) lines of
|
||||
form `beam_candidate_text \t score`. This file can be generated by `scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/ngram_lm/eval_beamsearch_ngram.py>`__
|
||||
|
||||
#. Rescore the candidates by `scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py <https://github.com/NVIDIA/NeMo/blob/main/scripts/asr_language_modeling/neural_rescorer/eval_neural_rescorer.py>`__.
|
||||
|
||||
.. code::
|
||||
python eval_neural_rescorer.py
|
||||
--lm_model=[path to .nemo file of the LM]
|
||||
--beams_file=[path to beams .tsv file]
|
||||
--beam_size=[size of the beams]
|
||||
--eval_manifest=[path to eval manifest .json file]
|
||||
--batch_size=[batch size used for inference on the LM model]
|
||||
--alpha=[the value for the parameter rescorer_alpha]
|
||||
--beta=[the value for the parameter rescorer_beta]
|
||||
--scores_output_file=[the optional path to store the rescored candidates]
|
||||
|
||||
The candidates along with their new scores would be stored at the file specified by `--scores_output_file`.
|
||||
|
||||
The following is the list of the arguments for the evaluation script:
|
||||
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| **Argument** |**Type**| **Default** | **Description** |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| lm_model | str | Required | The path of the '.nemo' file of the ASR model to extract the tokenizer. |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| eval_manifest | str | Required | Path to the evaluation manifest file (.json manifest file) |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beams_file | str | Required | path to beams file (.tsv) containing the candidates and their scores |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beam_size | int | Required | The width of the beams (number of candidates) generated by the decoder |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| alpha | float | None | The value for parameter rescorer_alpha |
|
||||
| | | | Not passing value would enable linear search for rescorer_alpha |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| beta | float | None | The value for parameter rescorer_beta |
|
||||
| | | | Not passing value would enable linear search for rescorer_beta |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| batch_size | int | 16 | The batch size used to calculate the scores |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| scores_output_file | str | None | The optional file to store the rescored beams |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| use_amp | bool | ``False`` | Whether to use AMP if available calculate the scores |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
| device | str | cuda | The device to load LM model onto to calculate the scores |
|
||||
| | | | It can be 'cpu', 'cuda', 'cuda:0', 'cuda:1', ... |
|
||||
+---------------------+--------+------------------+-------------------------------------------------------------------------+
|
||||
|
||||
|
||||
Hyperparameter Linear Search
|
||||
----------------------------
|
||||
|
||||
This script also supports linear search for parameters `rescorer_alpha` and `rescorer_beta`. If any of the two is not
|
||||
provided, a linear search is performed to find the best value for that parameter. When linear search is used, initially
|
||||
`rescorer_beta` is set to zero and the best value for `rescorer_alpha` is found, then `rescorer_alpha` is fixed with
|
||||
that value and another linear search is done to find the best value for `rescorer_beta`.
|
||||
If any of the of these two parameters is already specified, then search for that one is skipped. After each search for a
|
||||
parameter, the plot of WER% for different values of the parameter is also shown.
|
||||
|
||||
It is recommended to first use the linear search for both parameters on a validation set by not providing any values for `--alpha` and `--beta`.
|
||||
Then check the WER curves and decide on the best values for each parameter. Finally, evaluate the best values on the test set.
|
||||
@@ -0,0 +1,528 @@
|
||||
NeMo ASR Configuration Files
|
||||
============================
|
||||
|
||||
This section describes the NeMo configuration file setup that is specific to models in the ASR collection. For general information
|
||||
about how to set up and run experiments that is common to all NeMo models (e.g. Experiment Manager and PyTorch Lightning trainer
|
||||
parameters), see the :doc:`../core/core` section.
|
||||
|
||||
The model section of the NeMo ASR configuration files generally requires information about the dataset(s) being used, the preprocessor
|
||||
for audio files, parameters for any augmentation being performed, as well as the model architecture specification. The sections on
|
||||
this page cover each of these in more detail.
|
||||
|
||||
Example configuration files for all of the NeMo ASR scripts can be found in the
|
||||
`config directory of the examples <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples/asr/conf>`_.
|
||||
|
||||
|
||||
Dataset Configuration
|
||||
---------------------
|
||||
|
||||
Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and
|
||||
``test_ds`` sections in the configuration file, respectively. Depending on the task, there may be arguments specifying the sample rate
|
||||
of the audio files, the vocabulary of the dataset (for character prediction), whether or not to shuffle the dataset, and so on. You may
|
||||
also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command-line at runtime.
|
||||
|
||||
Any initialization parameter that is accepted for the Dataset class used in the experiment can be set in the config file.
|
||||
Refer to the `Datasets <./api.html#Datasets>`__ section of the API for a list of Datasets and their respective parameters.
|
||||
|
||||
An example ASR train and validation configuration should look similar to the following:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# Specified at the beginning of the config file
|
||||
labels: &labels [" ", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
|
||||
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "'"]
|
||||
|
||||
model:
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: *labels # Uses the labels above
|
||||
batch_size: 32
|
||||
trim_silence: True
|
||||
max_duration: 16.7
|
||||
shuffle: True
|
||||
is_tarred: False # If set to true, uses the tarred version of the Dataset
|
||||
tarred_audio_filepaths: null # Not used if is_tarred is false
|
||||
tarred_shard_strategy: "scatter" # Not used if is_tarred is false
|
||||
num_workers: 8
|
||||
pin_memory: true
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: *labels # Uses the labels above
|
||||
batch_size: 32
|
||||
shuffle: False # No need to shuffle the validation data
|
||||
num_workers: 8
|
||||
pin_memory: true
|
||||
|
||||
|
||||
Preprocessor Configuration
|
||||
--------------------------
|
||||
|
||||
If you are loading audio files for your experiment, you will likely want to use a preprocessor to convert from the
|
||||
raw audio signal to features (e.g. mel-spectrogram or MFCC). The ``preprocessor`` section of the config specifies the audio
|
||||
preprocessor to be used via the ``_target_`` field, as well as any initialization parameters for that preprocessor.
|
||||
|
||||
An example of specifying a preprocessor is as follows:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
preprocessor:
|
||||
# _target_ is the audio preprocessor module you want to use
|
||||
_target_: nemo.collections.asr.modules.AudioToMelSpectrogramPreprocessor
|
||||
normalize: "per_feature"
|
||||
window_size: 0.02
|
||||
...
|
||||
# Other parameters for the preprocessor
|
||||
|
||||
Refer to the `Audio Preprocessors <./api.html#Audio Preprocessors>`__ API section for the preprocessor options, expected arguments,
|
||||
and defaults.
|
||||
|
||||
Augmentation Configurations
|
||||
---------------------------
|
||||
|
||||
There are a few on-the-fly spectrogram augmentation options for NeMo ASR, which can be specified by the
|
||||
configuration file using a ``spec_augment`` section.
|
||||
|
||||
For example, there are options for `Cutout <https://arxiv.org/abs/1708.04552>`_ and
|
||||
`SpecAugment <https://arxiv.org/abs/1904.08779>`_ available via the ``SpectrogramAugmentation`` module.
|
||||
|
||||
The following example sets up both ``Cutout`` (via the ``rect_*`` parameters) and ``SpecAugment`` (via the ``freq_*``
|
||||
and ``time_*`` parameters).
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
spec_augment:
|
||||
_target_: nemo.collections.asr.modules.SpectrogramAugmentation
|
||||
# Cutout parameters
|
||||
rect_masks: 5 # Number of rectangles to cut from any given spectrogram
|
||||
rect_freq: 50 # Max cut of size 50 along the frequency dimension
|
||||
rect_time: 120 # Max cut of size 120 along the time dimension
|
||||
# SpecAugment parameters
|
||||
freq_masks: 2 # Cut two frequency bands
|
||||
freq_width: 15 # ... of width 15 at maximum
|
||||
time_masks: 5 # Cut out 10 time bands
|
||||
time_width: 25 # ... of width 25 at maximum
|
||||
|
||||
You can use any combination of ``Cutout``, frequency/time ``SpecAugment``, or neither of them.
|
||||
|
||||
With NeMo ASR, you can also add augmentation pipelines that can be used to simulate various kinds of noise
|
||||
added to audio in the channel. Augmentors in a pipeline are applied on the audio data read in the data layer. Online
|
||||
augmentors can be specified in the config file using an ``augmentor`` section in ``train_ds``. The following example
|
||||
adds an augmentation pipeline that first adds white noise to an audio sample with a probability of 0.5 and at a level
|
||||
randomly picked between -50 dB and -10 dB and then passes the resultant samples through a room impulse response randomly
|
||||
picked from the manifest file provided for ``impulse`` augmentation in the config file.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
train_ds:
|
||||
...
|
||||
augmentor:
|
||||
white_noise:
|
||||
prob: 0.5
|
||||
min_level: -50
|
||||
max_level: -10
|
||||
impulse:
|
||||
prob: 0.3
|
||||
manifest_path: /path/to/impulse_manifest.json
|
||||
|
||||
Refer to the `Audio Augmentors <./api.html#Audio Augmentors>`__ API section for more details.
|
||||
|
||||
Tokenizer Configurations
|
||||
------------------------
|
||||
|
||||
Some models utilize sub-word encoding via an external tokenizer instead of explicitly defining their vocabulary.
|
||||
|
||||
For such models, a ``tokenizer`` section is added to the model config. ASR models currently support two types of
|
||||
custom tokenizers:
|
||||
|
||||
- Google Sentencepiece tokenizers (tokenizer type of ``bpe`` in the config)
|
||||
- HuggingFace WordPiece tokenizers (tokenizer type of ``wpe`` in the config)
|
||||
|
||||
In order to build custom tokenizers, refer to the ``ASR_with_Subword_Tokenization`` notebook available in the
|
||||
ASR tutorials directory.
|
||||
|
||||
The following example sets up a ``SentencePiece Tokenizer`` at a path specified by the user:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
tokenizer:
|
||||
dir: "<path to the directory that contains the custom tokenizer files>"
|
||||
type: "bpe" # can be "bpe" or "wpe"
|
||||
|
||||
For models which utilize sub-word tokenization, we share the decoder module (``ConvASRDecoder``) with character tokenization models.
|
||||
All parameters are shared, but for models which utilize sub-word encoding, there are minor differences when setting up the config. For
|
||||
such models, the tokenizer is utilized to fill in the missing information when the model is constructed automatically.
|
||||
|
||||
For example, a decoder config corresponding to a sub-word tokenization model should look similar to the following:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASRDecoder
|
||||
feat_in: *enc_final
|
||||
num_classes: -1 # filled with vocabulary size from tokenizer at runtime
|
||||
vocabulary: [] # filled with vocabulary from tokenizer at runtime
|
||||
|
||||
|
||||
Model Architecture Configurations
|
||||
---------------------------------
|
||||
|
||||
Each configuration file should describe the model architecture being used for the experiment. Models in the NeMo ASR collection need
|
||||
an ``encoder`` section and a ``decoder`` section, with the ``_target_`` field specifying the module to use for each.
|
||||
|
||||
Here is the list of the parameters in the model section which are shared among most of the ASR models:
|
||||
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
| **Parameter** | **Datatype** | **Description** | **Supported Values** |
|
||||
+=========================+==================+===============================================================================================================+=================================+
|
||||
| :code:`log_prediction` | bool | Whether a random sample should be printed in the output at each step, along with its predicted transcript. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
| :code:`ctc_reduction` | string | Specifies the reduction type of CTC loss. Defaults to ``mean_batch`` which would take the average over the | :code:`none`, |
|
||||
| | | batch after taking the average over the length of each sample. | :code:`mean_batch` |
|
||||
| | | | :code:`mean`, :code:`sum` |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
|
||||
The following sections go into more detail about the specific configurations of each model architecture.
|
||||
|
||||
For more information about the ASR models, refer to the :doc:`Models <./models>` section.
|
||||
|
||||
Jasper and QuartzNet
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The `Jasper <./models.html#Jasper>`__ and `QuartzNet <./models.html#QuartzNet>`__ models are very similar, and as such the components in their
|
||||
configs are very similar as well.
|
||||
|
||||
Both architectures use the ``ConvASREncoder`` for the ``encoder``, with parameters detailed in the table below. The encoder parameters
|
||||
include details about the Jasper/QuartzNet ``[BxR]`` encoder architecture, including how many blocks to use (``B``), how many times
|
||||
to repeat each sub-block (``R``), and the convolution parameters for each block.
|
||||
|
||||
The number of blocks ``B`` is determined by the number of list elements under ``jasper`` minus the one prologue and two epilogue blocks.
|
||||
The number of sub-blocks ``R`` is determined by setting the ``repeat`` parameter.
|
||||
|
||||
To use QuartzNet (which uses more compact time-channel separable convolutions) instead of Jasper, add :code:`separable: true` to all
|
||||
but the last block in the architecture.
|
||||
|
||||
Change the parameter name ``jasper``.
|
||||
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+
|
||||
| **Parameter** | **Datatype** | **Description** | **Supported Values** |
|
||||
+=========================+==================+===============================================================================================================+=====================================+
|
||||
| :code:`feat_in` | int | The number of input features. Should be equal to :code:`features` in the preprocessor parameters. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+
|
||||
| :code:`activation` | string | Which activation function to use in the encoder. | :code:`hardtanh`, :code:`relu`, |
|
||||
| | | | :code:`selu`, :code:`swish` |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+
|
||||
| :code:`conv_mask` | bool | Whether to use masked convolutions in the encoder. Defaults to ``true``. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+
|
||||
| :code:`jasper` | | A list of blocks that specifies your encoder architecture. Each entry in this list represents one block in | |
|
||||
| | | the architecture and contains the parameters for that block, including convolution parameters, dropout, and | |
|
||||
| | | the number of times the block is repeated. Refer to the `Jasper <https://arxiv.org/pdf/1904.03288.pdf>`_ and | |
|
||||
| | | `QuartzNet <https://arxiv.org/pdf/1910.10261.pdf>`_ papers for details about specific model configurations. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+-------------------------------------+
|
||||
|
||||
A QuartzNet 15x5 (fifteen blocks, each sub-block repeated five times) encoder configuration should look similar to the following example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# Specified at the beginning of the file for convenience
|
||||
n_mels: &n_mels 64 # Used for both the preprocessor and encoder as number of input features
|
||||
repeat: &repeat 5 # R=5
|
||||
dropout: &dropout 0.0
|
||||
separable: &separable true # Set to true for QN. Set to false for Jasper.
|
||||
|
||||
model:
|
||||
...
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASREncoder
|
||||
feat_in: *n_mels # Should match "features" in the preprocessor.
|
||||
activation: relu
|
||||
conv_mask: true
|
||||
|
||||
jasper: # This field name should be "jasper" for both types of models.
|
||||
|
||||
# Prologue block
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: 1 # Prologue block is not repeated.
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [2]
|
||||
|
||||
# Block 1
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 256
|
||||
kernel: [33]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
... # Entries for blocks 2~14
|
||||
|
||||
# Block 15
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [75]
|
||||
repeat: *repeat
|
||||
residual: true
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
# Two epilogue blocks
|
||||
- dilation: [2]
|
||||
dropout: *dropout
|
||||
filters: 512
|
||||
kernel: [87]
|
||||
repeat: 1 # Epilogue blocks are not repeated
|
||||
residual: false
|
||||
separable: *separable
|
||||
stride: [1]
|
||||
|
||||
- dilation: [1]
|
||||
dropout: *dropout
|
||||
filters: &enc_filters 1024
|
||||
kernel: [1]
|
||||
repeat: 1 # Epilogue blocks are not repeated
|
||||
residual: false
|
||||
stride: [1]
|
||||
|
||||
Both Jasper and QuartzNet use the ``ConvASRDecoder`` as the decoder. The decoder parameters are detailed in the following table.
|
||||
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
| **Parameter** | **Datatype** | **Description** | **Supported Values** |
|
||||
+=========================+==================+===============================================================================================================+=================================+
|
||||
| :code:`feat_in` | int | The number of input features to the decoder. Should be equal to the number of filters in the last block of | |
|
||||
| | | the encoder. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
| :code:`vocabulary` | list | A list of the valid output characters for your model. For example, for an English dataset, this could be a | |
|
||||
| | | list of all lowercase letters, space, and apostrophe. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
| :code:`num_classes` | int | Number of output classes, i.e. the length of :code:`vocabulary`. | |
|
||||
+-------------------------+------------------+---------------------------------------------------------------------------------------------------------------+---------------------------------+
|
||||
|
||||
For example, a decoder config corresponding to the encoder above should look similar to the following:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASRDecoder
|
||||
feat_in: *enc_filters
|
||||
vocabulary: *labels
|
||||
num_classes: 28 # Length of the vocabulary list
|
||||
|
||||
Citrinet
|
||||
~~~~~~~~
|
||||
|
||||
The `Citrinet <./models.html#Citrinet>`__ and `QuartzNet <./models.html#QuartzNet>`__ models are very similar, and as such the
|
||||
components in their configs are very similar as well. Citrinet utilizes Squeeze and Excitation, as well as sub-word tokenization, in
|
||||
contrast to QuartzNet. Depending on the dataset, we utilize different tokenizers. For Librispeech, we utilize the HuggingFace WordPiece
|
||||
tokenizer, and for all other datasets we utilize the Google Sentencepiece tokenizer - usually the ``unigram`` tokenizer type.
|
||||
|
||||
Both architectures use the ``ConvASREncoder`` for the ``encoder``, with parameters detailed above. The encoder parameters include
|
||||
details about the Citrinet-C encoder architecture, including how many filters are used per channel (``C``). The Citrinet-C
|
||||
configuration is a shortform notation for Citrinet-21x5xC, such that ``B = 21`` and ``R = 5`` are the default and should generally
|
||||
not be changed.
|
||||
|
||||
To use Citrinet instead of QuartzNet, refer to the ``citrinet_512.yaml`` configuration found inside the ``examples/asr/conf/citrinet``
|
||||
directory. Citrinet is primarily comprised of the same :class:`~nemo.collections.asr.parts.submodules.jasper.JasperBlock` as ``Jasper`` or
|
||||
``QuartzNet`.
|
||||
|
||||
While the configs for Citrinet and QuartzNet are similar, we note the additional flags used for Citrinet below. Refer to the
|
||||
``JasperBlock`` documentation for the meaning of these arguments.
|
||||
|
||||
+---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+
|
||||
| **Parameter** | **Datatype** | **Description** | **Supported Values** |
|
||||
+===========================+==================+===========================================================================================================+===================================+
|
||||
| :code:`se` | bool | Whether to apply squeeze-and-excitation mechanism or not. | :code:`true` or :code:`false` |
|
||||
+---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+
|
||||
| :code:`se_context_size` | int | SE context size. -1 means global context. | :code:`-1` or :code:`+ve int` |
|
||||
+---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+
|
||||
| :code:`stride_last` | bool | Stride on the final repeated block or all repeated blocks. | :code:`true` or :code:`false` |
|
||||
+---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+
|
||||
| :code:`residual_mode` | str | Type of residual branch to construct. | :code:`"add"` or |
|
||||
| | | Can be pointwise residual addition or pointwise strided residual attention | :code:`"stride_add"` |
|
||||
+---------------------------+------------------+-----------------------------------------------------------------------------------------------------------+-----------------------------------+
|
||||
|
||||
A Citrinet-512 config should look similar to the following:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
# Specify some defaults across the entire model
|
||||
model_defaults:
|
||||
repeat: 5
|
||||
dropout: 0.1
|
||||
separable: true
|
||||
se: true
|
||||
se_context_size: -1
|
||||
...
|
||||
encoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASREncoder
|
||||
feat_in: *n_mels # Should match "features" in the preprocessor.
|
||||
activation: relu
|
||||
conv_mask: true
|
||||
|
||||
jasper: # This field name should be "jasper" for the JasperBlock (which constructs Citrinet).
|
||||
|
||||
# Prologue block
|
||||
- filters: 512
|
||||
repeat: 1
|
||||
kernel: [5]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
# Block 1
|
||||
- filters: 512
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [11]
|
||||
stride: [2]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
stride_last: true
|
||||
residual_mode: "stride_add"
|
||||
|
||||
... # Entries for blocks 2~21
|
||||
|
||||
# Block 22
|
||||
- filters: 512
|
||||
repeat: ${model.model_defaults.repeat}
|
||||
kernel: [39]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: ${model.model_defaults.dropout}
|
||||
residual: true
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
# Epilogue block
|
||||
|
||||
- filters: &enc_final 640
|
||||
repeat: 1
|
||||
kernel: [41]
|
||||
stride: [1]
|
||||
dilation: [1]
|
||||
dropout: 0.0
|
||||
residual: false
|
||||
separable: ${model.model_defaults.separable}
|
||||
se: ${model.model_defaults.se}
|
||||
se_context_size: ${model.model_defaults.se_context_size}
|
||||
|
||||
As mentioned above, Citrinet uses the ``ConvASRDecoder`` as the decoder layer similar to QuartzNet. Only the configuration must be
|
||||
changed slightly as Citrinet utilizes sub-word tokenization.
|
||||
|
||||
.. note::
|
||||
The following information is relevant to any of the above models that implements its encoder as an :class:`~nemo.collections.asr.modules.conv_asr.ConvASREncoder`, and utilizes the ``SqueezeExcite`` mechanism.
|
||||
|
||||
The ``SqueezeExcite`` block within a :class:`~nemo.collections.asr.modules.conv_asr.ConvASREncoder` network can be modified to utilize a different context window after the model has been instantiated (even after the model has been trained) so as to evaluate the model with limited context. This can be achieved using the :meth:`~nemo.collections.asr.parts.mixins.mixins.ASRModuleMixin.change_conv_asr_se_context_window`
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Here, model can be any model that has a `ConvASREncoder` as its encoder, and utilized `SqueezeExcite` blocks
|
||||
# `context_window` : It is an integer representing the number of timeframes (each corresponding to some window stride).
|
||||
# `update_config` : Bool flag which determines whether the config of the model should be updated to reflect the new context window.
|
||||
|
||||
# Here, we specify that 128 timeframes of 0.01s stride should be the context window
|
||||
# This is equivalent to 128 * 0.01s context window for `SqueezeExcite`
|
||||
model.change_conv_asr_se_context_window(context_window=128, update_config=True)
|
||||
|
||||
Conformer-CTC
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
The config files for Conformer-CTC model contain character-based encoding and sub-word encoding at
|
||||
``<NeMo_git_root>/examples/asr/conf/conformer/conformer_ctc_char.yaml`` and ``<NeMo_git_root>/examples/asr/conf/conformer/conformer_ctc_bpe.yaml``
|
||||
respectively. Some components of the configs of `Conformer-CTC <./models.html#Conformer-CTC>`__ include the following datasets:
|
||||
|
||||
- ``train_ds``, ``validation_ds``, and ``test_ds``
|
||||
- opimizer (``optim``)
|
||||
- augmentation (``spec_augment``)
|
||||
- ``decoder``
|
||||
- ``trainer``
|
||||
``exp_manager``
|
||||
|
||||
These datasets are similar to other ASR models like `QuartzNet <./models.html#QuartzNet>`__. There should be a tokenizer section where you can
|
||||
specify the tokenizer if you want to use sub-word encoding instead of character-based encoding.
|
||||
|
||||
The encoder section includes the details about the Conformer-CTC encoder architecture. You may find more information in the
|
||||
config files and also :doc:`nemo.collections.asr.modules.ConformerEncoder<./api.html#nemo.collections.asr.modules.ConformerEncoder>`.
|
||||
|
||||
|
||||
Fine-tuning Configurations
|
||||
-------------------------
|
||||
|
||||
All ASR scripts support easy fine-tuning by partially/fully loading the pretrained weights from a checkpoint into the currently instantiated model. Pre-trained weights can be provided in multiple ways -
|
||||
|
||||
1) Providing a path to a NeMo model (via ``init_from_nemo_model``)
|
||||
2) Providing a name of a pretrained NeMo model (which will be downloaded via the cloud) (via ``init_from_pretrained_model``)
|
||||
3) Providing a path to a Pytorch Lightning checkpoint file (via ``init_from_ptl_ckpt``)
|
||||
|
||||
Fine-tuning via a NeMo model
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
python examples/asr/script_to_<script_name>.py \
|
||||
--config-path=<path to dir of configs> \
|
||||
--config-name=<name of config without .yaml>) \
|
||||
model.train_ds.manifest_filepath="<path to manifest file>" \
|
||||
model.validation_ds.manifest_filepath="<path to manifest file>" \
|
||||
trainer.gpus=-1 \
|
||||
trainer.max_epochs=50 \
|
||||
+init_from_nemo_model="<path to .nemo model file>"
|
||||
|
||||
|
||||
Fine-tuning via a NeMo pretrained model name
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
python examples/asr/script_to_<script_name>.py \
|
||||
--config-path=<path to dir of configs> \
|
||||
--config-name=<name of config without .yaml>) \
|
||||
model.train_ds.manifest_filepath="<path to manifest file>" \
|
||||
model.validation_ds.manifest_filepath="<path to manifest file>" \
|
||||
trainer.gpus=-1 \
|
||||
trainer.max_epochs=50 \
|
||||
+init_from_pretrained_model="<name of pretrained checkpoint>"
|
||||
|
||||
Fine-tuning via a Pytorch Lightning checkpoint
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
python examples/asr/script_to_<script_name>.py \
|
||||
--config-path=<path to dir of configs> \
|
||||
--config-name=<name of config without .yaml>) \
|
||||
model.train_ds.manifest_filepath="<path to manifest file>" \
|
||||
model.validation_ds.manifest_filepath="<path to manifest file>" \
|
||||
trainer.gpus=-1 \
|
||||
trainer.max_epochs=50 \
|
||||
+init_from_ptl_ckpt="<name of pytorch lightning checkpoint>"
|
||||
@@ -0,0 +1,21 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
QuartzNet15x5Base-En,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels"
|
||||
stt_en_jasper10x5dr,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_jasper10x5dr"
|
||||
stt_en_citrinet_256,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_256"
|
||||
stt_en_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_512"
|
||||
stt_en_citrinet_1024,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_1024"
|
||||
stt_ca_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5"
|
||||
stt_it_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_it_quartznet15x5"
|
||||
stt_fr_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_fr_quartznet15x5"
|
||||
stt_es_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_quartznet15x5"
|
||||
stt_de_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_de_quartznet15x5"
|
||||
stt_pl_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_pl_quartznet15x5"
|
||||
stt_ru_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ru_quartznet15x5"
|
||||
stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5"
|
||||
stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512"
|
||||
stt_en_conformer_ctc_small,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small"
|
||||
stt_en_conformer_ctc_medium,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium"
|
||||
stt_en_conformer_ctc_large,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large"
|
||||
stt_en_conformer_ctc_small_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small_ls"
|
||||
stt_en_conformer_ctc_medium_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium_ls"
|
||||
stt_en_conformer_ctc_large_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls"
|
||||
|
@@ -0,0 +1,2 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
asrlm_en_transformer_large_ls,TransformerLMModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:asrlm_en_transformer_large_ls"
|
||||
|
@@ -0,0 +1,2 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_ca_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ca_quartznet15x5"
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_de_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_de_quartznet15x5"
|
||||
|
||||
|
@@ -0,0 +1,14 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
QuartzNet15x5Base-En,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels"
|
||||
stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5"
|
||||
stt_en_jasper10x5dr,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_jasper10x5dr"
|
||||
stt_en_citrinet_256,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_256"
|
||||
stt_en_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_512"
|
||||
stt_en_citrinet_1024,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_citrinet_1024"
|
||||
stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512"
|
||||
stt_en_conformer_ctc_small,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small"
|
||||
stt_en_conformer_ctc_medium,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium"
|
||||
stt_en_conformer_ctc_large,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large"
|
||||
stt_en_conformer_ctc_small_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_small_ls"
|
||||
stt_en_conformer_ctc_medium_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_medium_ls"
|
||||
stt_en_conformer_ctc_large_ls,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_en_conformer_ctc_large_ls"
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_es_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_es_quartznet15x5"
|
||||
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_fr_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_fr_quartznet15x5"
|
||||
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_it_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_it_quartznet15x5"
|
||||
|
||||
|
@@ -0,0 +1,2 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_pl_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_pl_quartznet15x5"
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_ru_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_ru_quartznet15x5"
|
||||
|
||||
|
@@ -0,0 +1,3 @@
|
||||
Model,Model Base Class,Model Card
|
||||
stt_zh_quartznet15x5,EncDecCTCModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_quartznet15x5"
|
||||
stt_zh_citrinet_512,EncDecCTCBPEModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:stt_zh_citrinet_512"
|
||||
|
@@ -0,0 +1,268 @@
|
||||
Datasets
|
||||
========
|
||||
|
||||
NeMo has scripts to convert several common ASR datasets into the format expected by the ``nemo_asr`` collection. You can get started
|
||||
with those datasets by following the instructions to run those scripts in the section appropriate to each dataset below.
|
||||
|
||||
If the user has their own data and want to preprocess it to use with NeMo ASR models, refer to the `Preparing Custom ASR Data`_ section.
|
||||
|
||||
If the user already has a dataset that you want to convert to a tarred format, refer to the `Tarred Datasets`_ section.
|
||||
|
||||
.. _LibriSpeech_dataset:
|
||||
|
||||
LibriSpeech
|
||||
-----------
|
||||
|
||||
Run the following scripts to download the LibriSpeech data and convert it into the format expected by `nemo_asr`. At least 250GB free
|
||||
space is required.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# install sox
|
||||
sudo apt-get install sox
|
||||
mkdir data
|
||||
python get_librispeech_data.py --data_root=data --data_set=ALL
|
||||
|
||||
After this, the ``data`` folder should contain wav files and ``.json`` manifests for NeMo ASR datalayer.
|
||||
|
||||
Each line is a training example. ``audio_filepath`` contains the path to the wav file, ``duration`` is the duration in seconds, and ``text`` is the transcript:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{"audio_filepath": "<absolute_path_to>/1355-39947-0000.wav", "duration": 11.3, "text": "psychotherapy and the community both the physician and the patient find their place in the community the life interests of which are superior to the interests of the individual"}
|
||||
{"audio_filepath": "<absolute_path_to>/1355-39947-0001.wav", "duration": 15.905, "text": "it is an unavoidable question how far from the higher point of view of the social mind the psychotherapeutic efforts should be encouraged or suppressed are there any conditions which suggest suspicion of or direct opposition to such curative work"}
|
||||
|
||||
Fisher English Training Speech
|
||||
------------------------------
|
||||
|
||||
Run these scripts to convert the Fisher English Training Speech data into a format expected by the ``nemo_asr`` collection.
|
||||
|
||||
In brief, the following scripts convert the ``.sph`` files to ``.wav``, slices those files into smaller audio samples, matches the
|
||||
smaller slices with their corresponding transcripts, and splits the resulting audio segments into train, validation, and test sets
|
||||
(with one manifest each).
|
||||
|
||||
.. note::
|
||||
- 106 GB of space is required to run the ``.wav`` conversion
|
||||
- additional 105 GB is required for the slicing and matching
|
||||
- ``sph2pipe`` is required in order to run the ``.wav`` conversion
|
||||
|
||||
**Instructions**
|
||||
|
||||
The following scripts assume that you already have the Fisher dataset from the Linguistic Data Consortium, with a directory structure
|
||||
that looks similar to the following:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
FisherEnglishTrainingSpeech/
|
||||
├── LDC2004S13-Part1
|
||||
│ ├── fe_03_p1_transcripts
|
||||
│ ├── fisher_eng_tr_sp_d1
|
||||
│ ├── fisher_eng_tr_sp_d2
|
||||
│ ├── fisher_eng_tr_sp_d3
|
||||
│ └── ...
|
||||
└── LDC2005S13-Part2
|
||||
├── fe_03_p2_transcripts
|
||||
├── fe_03_p2_sph1
|
||||
├── fe_03_p2_sph2
|
||||
├── fe_03_p2_sph3
|
||||
└── ...
|
||||
|
||||
The transcripts that will be used are located in the ``fe_03_p<1,2>_transcripts/data/trans`` directory. The audio files (``.sph``)
|
||||
are located in the remaining directories in an ``audio`` subdirectory.
|
||||
|
||||
#. Convert the audio files from ``.sph`` to ``.wav`` by running:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
cd <nemo_root>/scripts
|
||||
python fisher_audio_to_wav.py \
|
||||
--data_root=<fisher_root> --dest_root=<conversion_target_dir>
|
||||
|
||||
This will place the unsliced ``.wav`` files in ``<conversion_target_dir>/LDC200[4,5]S13-Part[1,2]/audio-wav/``. It will take several
|
||||
minutes to run.
|
||||
|
||||
#. Process the transcripts and slice the audio data.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python process_fisher_data.py \
|
||||
--audio_root=<conversion_target_dir> --transcript_root=<fisher_root> \
|
||||
--dest_root=<processing_target_dir> \
|
||||
--remove_noises
|
||||
|
||||
This script splits the full dataset into train, validation, test sets, and places the audio slices in the corresponding folders
|
||||
in the destination directory. One manifest is written out per set, which includes each slice's transcript, duration, and path.
|
||||
|
||||
This will likely take around 20 minutes to run. Once finished, delete the 10 minute long ``.wav`` files.
|
||||
|
||||
2000 HUB5 English Evaluation Speech
|
||||
-----------------------------------
|
||||
|
||||
Run the following script to convert the HUB5 data into a format expected by the ``nemo_asr`` collection.
|
||||
|
||||
Similarly, to the Fisher dataset processing scripts, this script converts the ``.sph`` files to ``.wav``, slices the audio files and
|
||||
transcripts into utterances, and combines them into segments of some minimum length (default is 10 seconds). The resulting segments
|
||||
are all written out to an audio directory and the corresponding transcripts are written to a manifest JSON file.
|
||||
|
||||
.. note::
|
||||
- 5 GB of free space is required to run this script
|
||||
- ``sph2pipe`` is also required to be installed
|
||||
|
||||
This script assumes you already have the 2000 HUB5 dataset from the Linguistic Data Consortium.
|
||||
|
||||
Run the following command to process the 2000 HUB5 English Evaluation Speech samples:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python process_hub5_data.py \
|
||||
--data_root=<path_to_HUB5_data> \
|
||||
--dest_root=<target_dir>
|
||||
|
||||
You can optionally include ``--min_slice_duration=<num_seconds>`` if you would like to change the minimum audio segment duration.
|
||||
|
||||
AN4 Dataset
|
||||
-----------
|
||||
|
||||
This is a small dataset recorded and distributed by Carnegie Mellon University. It consists of recordings of people spelling out
|
||||
addresses, names, etc. Information about this dataset can be found on the `official CMU site <http://www.speech.cs.cmu.edu/databases/an4/>`_.
|
||||
|
||||
#. `Download and extract the dataset <http://www.speech.cs.cmu.edu/databases/an4/an4_sphere.tar.gz>`_ (which is labeled "NIST's Sphere audio (.sph) format (64M)".
|
||||
|
||||
#. Convert the ``.sph`` files to ``.wav`` using sox, and build one training and one test manifest.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python process_an4_data.py --data_root=<path_to_extracted_data>
|
||||
|
||||
After the script finishes, the ``train_manifest.json`` and ``test_manifest.json`` can be found in the ``<data_root>/an4/`` directory.
|
||||
|
||||
Aishell-1
|
||||
---------
|
||||
|
||||
To download the Aishell-1 data and convert it into a format expected by ``nemo_asr``, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# install sox
|
||||
sudo apt-get install sox
|
||||
mkdir data
|
||||
python get_aishell_data.py --data_root=data
|
||||
|
||||
After the script finishes, the ``data`` folder should contain a ``data_aishell`` folder which contains a wav file, a transcript folder, and related ``.json`` and ``vocab.txt`` files.
|
||||
|
||||
Aishell-2
|
||||
---------
|
||||
|
||||
To process the AIShell-2 dataset, in the command below, set the data folder of AIShell-2 using ``--audio_folder`` and where to push
|
||||
these files using ``--dest_folder``. In order to generate files in the supported format of ``nemo_asr``, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python process_aishell2_data.py --audio_folder=<data directory> --dest_folder=<destination directory>
|
||||
|
||||
After the script finishes, the ``train.json``, ``dev.json``, ``test.json``, and ``vocab.txt`` files can be found in the ``dest_folder`` directory.
|
||||
|
||||
Preparing Custom ASR Data
|
||||
-------------------------
|
||||
|
||||
The ``nemo_asr`` collection expects each dataset to consist of a set of utterances in individual audio files plus
|
||||
a manifest that describes the dataset, with information about one utterance per line (``.json``).
|
||||
The audio files can be of any format supported by `Pydub <https://github.com/jiaaro/pydub>`_, though we recommend
|
||||
WAV files as they are the default and have been most thoroughly tested.
|
||||
|
||||
There should be one manifest file per dataset that will be passed in, therefore, if the user wants separate training and validation
|
||||
datasets, they should also have separate manifests. Otherwise, thay will be loading validation data with their training data and vice
|
||||
versa.
|
||||
|
||||
Each line of the manifest should be in the following format:
|
||||
|
||||
.. code::
|
||||
|
||||
{"audio_filepath": "/path/to/audio.wav", "text": "the transcription of the utterance", "duration": 23.147}
|
||||
|
||||
The :code:`audio_filepath` field should provide an absolute path to the ``.wav`` file corresponding to the utterance.
|
||||
The :code:`text` field should contain the full transcript for the utterance, and the :code:`duration` field should
|
||||
reflect the duration of the utterance in seconds.
|
||||
|
||||
Each entry in the manifest (describing one audio file) should be bordered by '{' and '}' and must
|
||||
be contained on one line. The fields that describe the file should be separated by commas, and have the form :code:`"field_name": value`,
|
||||
as shown above. There should be no extra lines in the manifest, i.e. there should be exactly as many lines in the manifest as
|
||||
there are audio files in the dataset.
|
||||
|
||||
Since the manifest specifies the path for each utterance, the audio files do not have to be located
|
||||
in the same directory as the manifest, or even in any specific directory structure.
|
||||
|
||||
Once there is a manifest that describes each audio file in the dataset, use the dataset by passing
|
||||
in the manifest file path in the experiment config file, e.g. as ``training_ds.manifest_filepath=<path/to/manifest,json>``.
|
||||
|
||||
Tarred Datasets
|
||||
---------------
|
||||
|
||||
If experiments are run on a cluster with datasets stored on a distributed file system, the user will likely
|
||||
want to avoid constantly reading multiple small files and would prefer tarring their audio files.
|
||||
There are tarred versions of some NeMo ASR dataset classes for this case, such as the ``TarredAudioToCharDataset``
|
||||
(corresponding to the ``AudioToCharDataset``) and the ``TarredAudioToBPEDataset`` (corresponding to the
|
||||
``AudioToBPEDataset``). The tarred audio dataset classes in NeMo use `WebDataset <https://github.com/tmbdev/webdataset>`_.
|
||||
|
||||
To use an existing tarred dataset instead of a non-tarred dataset, set ``is_tarred: true`` in
|
||||
the experiment config file. Then, pass in the paths to all of the audio tarballs in ``tarred_audio_filepaths``, either as a list
|
||||
of filepaths, e.g. ``['/data/shard1.tar', '/data/shard2.tar']``, or in a single brace-expandable string, e.g.
|
||||
``'/data/shard_{1..64}.tar'`` or ``'/data/shard__OP_1..64_CL_'`` (recommended, see note below).
|
||||
|
||||
.. note::
|
||||
For brace expansion, there may be cases where ``{x..y}`` syntax cannot be used due to shell interference. This occurs most commonly
|
||||
inside SLURM scripts. Therefore, we provide a few equivalent replacements. Supported opening braces (equivalent to ``{``) are ``(``,
|
||||
``[``, ``<`` and the special tag ``_OP_``. Supported closing braces (equivalent to ``}``) are ``)``, ``]``, ``>`` and the special
|
||||
tag ``_CL_``. For SLURM based tasks, we suggest the use of the special tags for ease of use.
|
||||
|
||||
As with non-tarred datasets, the manifest file should be passed in ``manifest_filepath``. The dataloader assumes that the length
|
||||
of the manifest after filtering is the correct size of the dataset for reporting training progress.
|
||||
|
||||
The ``tarred_shard_strategy`` field of the config file can be set if you have multiple shards and are running an experiment with
|
||||
multiple workers. It defaults to ``scatter``, which preallocates a set of shards per worker which do not change during runtime.
|
||||
|
||||
For more information about the individual tarred datasets and the parameters available, including shuffling options,
|
||||
see the corresponding class APIs in the `Datasets <./api.html#Datasets>`__ section.
|
||||
|
||||
.. warning::
|
||||
If using multiple workers, the number of shards should be divisible by the world size to ensure an even
|
||||
split among workers. If it is not divisible, logging will give a warning but training will proceed, but likely hang at the last epoch.
|
||||
In addition, if using distributed processing, each shard must have the same number of entries after filtering is
|
||||
applied such that each worker ends up with the same number of files. We currently do not check for this in any dataloader, but the user's
|
||||
program may hang if the shards are uneven.
|
||||
|
||||
Conversion to Tarred Datasets
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
You can easily convert your existing NeMo-compatible ASR datasets using the
|
||||
`conversion script here <https://github.com/NVIDIA/NeMo/blob/v1.0.2/scripts/speech_recognition/convert_to_tarred_audio_dataset.py>`_.
|
||||
|
||||
.. code::
|
||||
|
||||
python convert_to_tarred_audio_dataset.py \
|
||||
--manifest_path=<path to the manifest file> \
|
||||
--target_dir=<path to output directory> \
|
||||
--num_shards=<number of tarfiles that will contain the audio>
|
||||
--max_duration=<float representing maximum duration of audio samples> \
|
||||
--min_duration=<float representing minimum duration of audio samples> \
|
||||
--shuffle --shuffle_seed=0
|
||||
|
||||
This script shuffles the entries in the given manifest (if ``--shuffle`` is set, which we recommend), filter
|
||||
audio files according to ``min_duration`` and ``max_duration``, and tar the remaining audio files to the directory
|
||||
``--target_dir`` in ``n`` shards, along with separate manifest and metadata files.
|
||||
|
||||
The files in the target directory should look similar to the following:
|
||||
|
||||
.. code::
|
||||
|
||||
target_dir/
|
||||
├── audio_1.tar
|
||||
├── audio_2.tar
|
||||
├── ...
|
||||
├── metadata.yaml
|
||||
└── tarred_audio_manifest.json
|
||||
|
||||
Note that file structures are flattened such that all audio files are at the top level in each tarball. This ensures that
|
||||
filenames are unique in the tarred dataset and the filepaths do not contain "-sub" and forward slashes in each ``audio_filepath`` are
|
||||
simply converted to underscores. For example, a manifest entry for ``/data/directory1/file.wav`` would be ``_data_directory1_file.wav``
|
||||
in the tarred dataset manifest, and ``/data/directory2/file.wav`` would be converted to ``_data_directory2_file.wav``.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 135 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 103 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 273 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 257 KiB |
@@ -0,0 +1,59 @@
|
||||
Automatic Speech Recognition (ASR)
|
||||
==================================
|
||||
|
||||
ASR, or Automatic Speech Recognition, refers to the problem of getting a program to automatically transcribe spoken language
|
||||
(speech-to-text). Our goal is usually to have a model that minimizes the Word Error Rate (WER) metric when transcribing speech input.
|
||||
In other words, given some audio file (e.g. a WAV file) containing speech, how do we transform this into the corresponding text with
|
||||
as few errors as possible?
|
||||
|
||||
Traditional speech recognition takes a generative approach, modeling the full pipeline of how speech sounds are produced in order to
|
||||
evaluate a speech sample. We would start from a language model that encapsulates the most likely orderings of words that are generated
|
||||
(e.g. an n-gram model), to a pronunciation model for each word in that ordering (e.g. a pronunciation table), to an acoustic model that
|
||||
translates those pronunciations to audio waveforms (e.g. a Gaussian Mixture Model).
|
||||
|
||||
Then, if we receive some spoken input, our goal would be to find the most likely sequence of text that would result in the given audio
|
||||
according to our generative pipeline of models. Overall, with traditional speech recognition, we try to model ``Pr(audio|transcript)*Pr(transcript)``,
|
||||
and take the argmax of this over possible transcripts.
|
||||
|
||||
Over time, neural nets advanced to the point where each component of the traditional speech recognition model could be replaced by a
|
||||
neural model that had better performance and that had a greater potential for generalization. For example, we could replace an n-gram
|
||||
model with a neural language model, and replace a pronunciation table with a neural pronunciation model, and so on. However, each of
|
||||
these neural models need to be trained individually on different tasks, and errors in any model in the pipeline could throw off the
|
||||
whole prediction.
|
||||
|
||||
Thus, we can see the appeal of end-to-end ASR architectures: discriminative models that simply take an audio input and give a textual
|
||||
output, and in which all components of the architecture are trained together towards the same goal. The model's encoder would be
|
||||
akin to an acoustic model for extracting speech features, which can then be directly piped to a decoder which outputs text. If desired,
|
||||
we could integrate a language model that would improve our predictions, as well.
|
||||
|
||||
And the entire end-to-end ASR model can be trained at once--a much easier pipeline to handle!
|
||||
|
||||
The full documentation tree is as follows:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 8
|
||||
|
||||
models
|
||||
datasets
|
||||
asr_language_modeling
|
||||
results
|
||||
configs
|
||||
api
|
||||
|
||||
Resources and Documentation
|
||||
---------------------------
|
||||
|
||||
Hands-on speech recognition tutorial notebooks can be found under `the ASR tutorials folder <https://github.com/NVIDIA/NeMo/tree/v1.0.2/tutorials/asr/>`_.
|
||||
If you are a beginner to NeMo, consider trying out the `ASR with NeMo <https://github.com/NVIDIA/NeMo/tree/v1.0.2/tutorials/asr/01_ASR_with_NeMo.ipynb>`_ tutorial.
|
||||
This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab.
|
||||
|
||||
If you are looking for information about a particular ASR model, or would like to find out more about the model
|
||||
architectures available in the `nemo_asr` collection, refer to the :doc:`Models <./models>` section.
|
||||
|
||||
NeMo includes preprocessing scripts for several common ASR datasets. The :doc:`Datasets <./datasets>` section contains instructions on
|
||||
running those scripts. It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data.
|
||||
|
||||
Information about how to load model checkpoints (either local files or pretrained ones from NGC), as well as a list of the checkpoints
|
||||
available on NGC are located on the :doc:`Checkpoints <./results>` section.
|
||||
|
||||
Documentation regarding the configuration files specific to the ``nemo_asr`` models can be found on the :doc:`Configuration Files <./configs>` section.
|
||||
@@ -0,0 +1,97 @@
|
||||
Models
|
||||
======
|
||||
|
||||
This section gives a brief overview of the models that NeMo's ASR collection currently supports.
|
||||
|
||||
Each of these models can be used with the example ASR scripts (in the ``<NeMo_git_root>/examples/asr`` directory) by
|
||||
specifying the model architecture in the config file used. Examples of config files for each model can be found in
|
||||
the ``<NeMo_git_root>/examples/asr/conf`` directory.
|
||||
|
||||
For more information about the config files and how they should be structured, refer to the :doc:`./configs` section.
|
||||
|
||||
Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found in the :doc:`./results`
|
||||
section. You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets. The checkpoints section
|
||||
also contains benchmark results for the available ASR models.
|
||||
|
||||
.. _Jasper_model:
|
||||
|
||||
Jasper
|
||||
------
|
||||
|
||||
Jasper ("Just Another Speech Recognizer") :cite:`asr-models-li2019jasper` is a deep time delay neural network (TDNN) comprising of
|
||||
blocks of 1D-convolutional layers. The Jasper family of models are denoted as ``Jasper_[BxR]`` where ``B`` is the number of blocks
|
||||
and ``R`` is the number of convolutional sub-blocks within a block. Each sub-block contains a 1-D convolution, batch normalization,
|
||||
ReLU, and dropout:
|
||||
|
||||
.. image:: images/jasper_vertical.png
|
||||
:align: center
|
||||
:alt: japer model
|
||||
:scale: 50%
|
||||
|
||||
Jasper models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModel` class.
|
||||
|
||||
QuartzNet
|
||||
---------
|
||||
|
||||
QuartzNet :cite:`asr-models-kriman2019quartznet` is a version of Jasper :cite:`asr-models-li2019jasper` model with separable
|
||||
convolutions and larger filters. It can achieve performance similar to Jasper but with an order of magnitude fewer parameters.
|
||||
Similarly to Jasper, the QuartzNet family of models are denoted as ``QuartzNet_[BxR]`` where ``B`` is the number of blocks and ``R``
|
||||
is the number of convolutional sub-blocks within a block. Each sub-block contains a 1-D *separable* convolution, batch normalization,
|
||||
ReLU, and dropout:
|
||||
|
||||
.. image:: images/quartz_vertical.png
|
||||
:align: center
|
||||
:alt: quartznet model
|
||||
:scale: 40%
|
||||
|
||||
QuartzNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModel` class.
|
||||
|
||||
Citrinet
|
||||
--------
|
||||
|
||||
Citrinet is a version of QuartzNet :cite:`asr-models-kriman2019quartznet` that extends ContextNet :cite:`asr-models-han2020contextnet`,
|
||||
utilizing subword encoding (via Word Piece tokenization) and Squeeze-and-Excitation mechanism :cite:`asr-models-hu2018squeeze` to
|
||||
obtain highly accurate audio transcripts while utilizing a non-autoregressive CTC based decoding scheme for efficient inference.
|
||||
|
||||
.. image:: images/citrinet_vertical.png
|
||||
:align: center
|
||||
:alt: citrinet model
|
||||
:scale: 50%
|
||||
|
||||
Citrinet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModelBPE` class.
|
||||
|
||||
.. _Conformer-CTC_model:
|
||||
|
||||
Conformer-CTC
|
||||
-------------
|
||||
|
||||
Conformer-CTC is a CTC-based variant of the Conformer model introduced in :cite:`asr-models-gulati2020conformer`. Conformer-CTC has a
|
||||
similar encoder as the original Conformer but uses CTC loss and decoding instead of RNNT loss, which makes it a non-autoregressive model.
|
||||
We also drop the LSTM decoder and instead use a linear decoder on the top of the encoder. This model uses the combination of
|
||||
self-attention and convolution modules to achieve the best of the two approaches, the self-attention layers can learn the global
|
||||
interaction while the convolutions efficiently capture the local correlations. The self-attention modules support both regular
|
||||
self-attention with absolute positional encoding, and also Transformer-XL's self-attention with relative positional encodings.
|
||||
|
||||
Here is the overall architecture of the encoder of Conformer-CTC:
|
||||
|
||||
.. image:: images/conformer_ctc.png
|
||||
:align: center
|
||||
:alt: Conformer-CTC Model
|
||||
:scale: 50%
|
||||
|
||||
This model supports both the sub-word level and character level encodings. You can find more details on the config files for the
|
||||
Conformer-CTC models at `Conformer-CTC <./configs.html#conformer-ctc>`. The variant with sub-word encoding is a BPE-based model
|
||||
which can be instantiated using the :class:`~nemo.collections.asr.models.EncDecCTCModelBPE` class, while the
|
||||
character-based variant is based on :class:`~nemo.collections.asr.models.EncDecCTCModel`.
|
||||
|
||||
You may find the example config files of Conformer-CTC model with character-based encoding at
|
||||
``<NeMo_git_root>/examples/asr/conf/conformer/conformer_ctc_char.yaml`` and
|
||||
with sub-word encoding at ``<NeMo_git_root>/examples/asr/conf/conformer/conformer_ctc_bpe.yaml``.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: asr_all.bib
|
||||
:style: plain
|
||||
:labelprefix: ASR-MODELS
|
||||
:keyprefix: asr-models-
|
||||
@@ -0,0 +1,181 @@
|
||||
Checkpoints
|
||||
===========
|
||||
|
||||
There are two main ways to load pretrained checkpoints in NeMo:
|
||||
|
||||
* Using the :code:`restore_from()` method to load a local checkpoint file (``.nemo``), or
|
||||
* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC.
|
||||
|
||||
Refer to the following sections for instructions and examples for each.
|
||||
|
||||
Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning. For resuming an unfinished
|
||||
training experiment, use the Experiment Manager to do so by setting the ``resume_if_exists`` flag to ``True``.
|
||||
|
||||
Loading Local Checkpoints
|
||||
-------------------------
|
||||
|
||||
NeMo automatically saves checkpoints of a model that is trained in a ``.nemo`` format. Alternatively, to manually save the model at any
|
||||
point, issue :code:`model.save_to(<checkpoint_path>.nemo)`.
|
||||
|
||||
If there is a local ``.nemo`` checkpoint that you'd like to load, use the :code:`restore_from()` method:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.<MODEL_BASE_CLASS>.restore_from(restore_path="<path/to/checkpoint/file.nemo>")
|
||||
|
||||
Where the model base class is the ASR model class of the original checkpoint, or the general ``ASRModel`` class.
|
||||
|
||||
NGC Pretrained Checkpoints
|
||||
--------------------------
|
||||
|
||||
The ASR collection has checkpoints of several models trained on various datasets for a variety of tasks. These checkpoints are
|
||||
obtainable via NGC `NeMo Automatic Speech Recognition collection <https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels>`_.
|
||||
The model cards on NGC contain more information about each of the checkpoints available.
|
||||
|
||||
The tables below list the ASR models available from NGC. The models can be accessed via the :code:`from_pretrained()` method inside
|
||||
the ASR Model class. In general, you can load any of these models with code in the following format:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.ASRModel.from_pretrained(model_name="<MODEL_NAME>")
|
||||
|
||||
Where the model name is the value under "Model Name" entry in the tables below.
|
||||
|
||||
For example, to load the base English QuartzNet model for speech recognition, run:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model = nemo_asr.models.ASRModel.from_pretrained(model_name="QuartzNet15x5Base-En")
|
||||
|
||||
You can also call :code:`from_pretrained()` from the specific model class (such as :code:`EncDecCTCModel`
|
||||
for QuartzNet) if you need to access a specific model functionality.
|
||||
|
||||
If you would like to programatically list the models available for a particular base class, you can use the
|
||||
:code:`list_available_models()` method.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
nemo_asr.models.<MODEL_BASE_CLASS>.list_available_models()
|
||||
|
||||
Transcribing/Inference
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
To perform inference and transcribe a sample of speech after loading the model, use the ``transcribe()`` method:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model.transcribe(paths2audio_files=[list of audio files], batch_size=BATCH_SIZE, logprobs=False)
|
||||
|
||||
Setting the argument ``logprobs`` to ``True`` returns the log probabilities instead of transcriptions. For more information, see :doc:`./api.html#modules`.
|
||||
The audio files should be 16KHz monochannel wav files.
|
||||
|
||||
Fine-tuning on Different Datasets
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
There are multiple ASR tutorials provided in the :ref:`Tutorials <tutorials>` section. Most of these tutorials explain how to instantiate a pre-trained model, prepare the model for fine-tuning on some dataset (in the same language) as a demonstration.
|
||||
|
||||
|
||||
Automatic Speech Recognition Models
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. csv-table::
|
||||
:file: data/asr_results.csv
|
||||
:align: left
|
||||
:widths: 30, 30, 40
|
||||
:header-rows: 1
|
||||
|
||||
Language Models for ASR
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. csv-table::
|
||||
:file: data/asrlm_results.csv
|
||||
:align: left
|
||||
:widths: 30, 30, 40
|
||||
:header-rows: 1
|
||||
|
||||
|
||||
Speech Recognition (Languages)
|
||||
------------------------------
|
||||
|
||||
English
|
||||
^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_en.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
Mandarin
|
||||
^^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_zh.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
German
|
||||
^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_de.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
Polish
|
||||
^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_pl.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
Italian
|
||||
^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_it.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
Russian
|
||||
^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_ru.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
-----------------------------
|
||||
|
||||
Spanish
|
||||
^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_es.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
|
||||
-----------------------------
|
||||
|
||||
Catalan
|
||||
^^^^^^^
|
||||
.. csv-table::
|
||||
:file: data/benchmark_ca.csv
|
||||
:align: left
|
||||
:widths: 40, 10, 50
|
||||
:header-rows: 1
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
NeMo Speaker Diarization API
|
||||
=============================
|
||||
|
||||
|
||||
Model Classes
|
||||
-------------
|
||||
.. autoclass:: nemo.collections.asr.models.ClusteringDiarizer
|
||||
:show-inheritance:
|
||||
:members: init_speaker_model, set_vad_model, init_vad_model, setup_vad_test_data, setup_spkr_test_data, run_vad, extract_embeddings, path2audio_files_to_manifest, diarize, make_nemo_file_from_folder, save_to, unpack_nemo_file, restore_from
|
||||
|
||||
|
||||
Mixins
|
||||
------
|
||||
|
||||
.. autoclass:: nemo.collections.asr.parts.mixins.mixins.DiarizationMixin
|
||||
:show-inheritance:
|
||||
:members:
|
||||
@@ -0,0 +1,65 @@
|
||||
NeMo Speaker Diarization Configuration Files
|
||||
============================================
|
||||
|
||||
Since speaker diarization model here is not a fully-trainble End-to-End model but an inference pipeline, we use **diarizer** instead of **model** which is used in other tasks.
|
||||
|
||||
The diarizer section will generally require information about the dataset(s) being
|
||||
used, models used in this pipline, as well as inference related parameters such as postprocessing of each models.
|
||||
The sections on this page cover each of these in more detail.
|
||||
|
||||
|
||||
Example configuration files for speaker diarization can be found in the
|
||||
``<NeMo_git_root>/examples/speaker_recognition/conf/speaker_diarization.yaml>``
|
||||
|
||||
.. note::
|
||||
For model details and deep understanding about configs, finetuning, tuning threshold, and evaluation,
|
||||
please refer to ``<NeMo_git_root>/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb``;
|
||||
for other applications such as possible integration with ASR, have a look at ``<NeMo_git_root>/tutorials/speaker_recognition/ASR_with_SpeakerDiarization.ipynb``.
|
||||
|
||||
|
||||
Dataset Configuration
|
||||
-----------------------
|
||||
|
||||
In contrast to other ASR related tasks or models in NeMo, speaker diarization supported in NeMo is a modular inference pipeline.
|
||||
Datasets here denotes the data you would like to perform speaker diarization on.
|
||||
|
||||
An example Speaker Diarization dataset configuration could look like:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
diarizer:
|
||||
oracle_num_speakers: ??? # if number of speakers is known input it or pass null if not known. Accepts int or path to file containing uniq-id and num of speakers of that session
|
||||
max_num_speakers: 8 # max number of speakers for each recording. If oracle num speakers is passed this value is ignored
|
||||
out_dir: ???
|
||||
paths2audio_files: ??? # file containing paths to audio files for diarzation
|
||||
path2groundtruth_rttm_files: null # (Optional) file containing paths to rttm files (this can be passed if we need to calculate DER rate based on ground truth rttm files).
|
||||
...
|
||||
|
||||
.. note::
|
||||
We expect audio and the corresponding RTTM to have the same base name and the name should be unique.
|
||||
|
||||
|
||||
Diarizer Architecture Configurations
|
||||
-------------------------------------
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
diarizer:
|
||||
...
|
||||
vad:
|
||||
model_path: null #.nemo local model path or pretrained model name or none
|
||||
window_length_in_sec: 0.15
|
||||
shift_length_in_sec: 0.01
|
||||
threshold: 0.5 # tune threshold on dev set. Check <NeMo_git_root>/scripts/voice_activity_detection/vad_tune_threshold.py
|
||||
vad_decision_smoothing: True
|
||||
smoothing_params:
|
||||
method: "median"
|
||||
overlap: 0.875
|
||||
|
||||
speaker_embeddings:
|
||||
oracle_vad_manifest: null # leave it null if to perform diarization with above VAD model else path to manifest file genrerated as shown in Datasets section
|
||||
model_path: ??? #.nemo local model path or pretrained model name
|
||||
window_length_in_sec: 1.5 # for better performace try with 3 sec and 1.5 sec
|
||||
shift_length_in_sec: 0.75 # for better performace try with 1.5 sec and 0.75 sec
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
vad_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_marblenet"
|
||||
vad_telephony_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_telephony_marblenet"
|
||||
speakerdiarization_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerdiarization_speakernet"
|
||||
|
@@ -0,0 +1,72 @@
|
||||
Datasets
|
||||
========
|
||||
Check out page :doc:`Speech Classification Datasets <../speech_classification/datasets>` and :doc:`Speaker Recogniton Datasets <../speaker_recognition/datasets>`
|
||||
for preparing datasets for training and validating VAD and speaker embedding models respectively.
|
||||
|
||||
For Speaker Diarization inference, ``diarizer`` expects following arguments :
|
||||
|
||||
- paths2audio_files
|
||||
- oracle_num_speakers
|
||||
- path2groundtruth_rttm_files [Optional]
|
||||
- oracle_vad_manifest (if vad/rttm files are known)[Optional but suggested for good performace. This avoids FPR and TPR]
|
||||
|
||||
paths2audio_files:
|
||||
- a file containing absolute paths to audio files or list of paths to audio files.
|
||||
For generating a file that contains paths to audio files, you can simply use ``find`` bash command as shown below:
|
||||
.. code-block:: bash
|
||||
|
||||
find $PWD/{data_dir} -iname '*.wav' > path_to_audiofiles.txt
|
||||
|
||||
.. note::
|
||||
We expect audio and the corresponding RTTM files to have the same base name and the name should be unique (uniq-id).
|
||||
|
||||
oracle_num_speakers:
|
||||
- if number of speakers is known input it or pass null if not known. Accepts int or path to file containing uniq-id with num of speakers of that session
|
||||
|
||||
Sample line (uniq-id `iaaa` session has 2 unique speakers )::
|
||||
|
||||
iaaa 2
|
||||
|
||||
path2groundtruth_rttm_files [Optional]:
|
||||
- To evaluate the diarizer system with known rttm files, One needs to provide a txt file (path2groundtruth_rttm_files) for groundtruth label files.
|
||||
use above mentioned ``find`` command to get all reference rttm files (use '*.rttm' as search pattern)
|
||||
|
||||
Each groundtruth label file should be in NIST Rich Transcription Time Marked (RTTM) format. Take one line from a RTTM file for example:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
SPEAKER TS3012d.Mix-Headset 1 331.573 0.671 <NA> <NA> MTD046ID <NA> <NA>
|
||||
|
||||
|
||||
oracle_vad_manifest [Optional but suggested for good performace. This reduces FPR and TPR]:
|
||||
- To perform just oracle diarization, that is taking speech activity time stamps from groundtruth RTTMs or another VAD from wild instead of NeMo VAD output, ``diarizer`` expects an oracle manifest json file that contains paths to audio files with offset for start time and duration of segment.
|
||||
|
||||
To prepare an oracle manifest file, use the script from ``scripts`` folder as shown below:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python $NeMo/scripts/ --paths2rttm_files=<path2groundtruth_rttm_files.txt> --paths2audio_files=<paths2audio_files.txt> --manifest_file=<output_oracle_manifest_file.json>
|
||||
|
||||
Here ``paths2audio_files`` and ``path2groundtruth_rttm_files`` are files containing paths to audio files as shown above.
|
||||
|
||||
AMI Meeting Corpus
|
||||
------------------
|
||||
|
||||
The following are the suggested parameters for getting Speaker Error Rate (SER) of 4.1% on AMI Lapel test set corpus:
|
||||
- diarizer.oracle_num_speakers = 4 (since there are exactly 4 speakers per each Lapel test set session)
|
||||
- diarizer.speaker_embeddings.model_path = ``speakerverification_speakernet`` (This model is trained on voxceleb dataset. ``Use this model for simialr non-telephonic speech datasets``)
|
||||
- diarizer.speaker_embeddings.window_length_in_sec = 3
|
||||
- diarizer.speaker_embeddings.shift_length_in_sec = 1.5
|
||||
|
||||
Input paths2audio_files, paths2rttm_files and oracle_vad_manifest by following steps as shown above
|
||||
|
||||
CallHome LDC97S42 (CH109)
|
||||
-------------------------
|
||||
|
||||
The following are the suggested parameters for getting Speaker Error Rate (SER) of 5.4% on CH109 set:
|
||||
- diarizer.oracle_num_speakers = 2 (since there are exactly 2 speakers per each ch109 session)
|
||||
- diarizer.speaker_embeddings.model_path = ``speakerdiarization_speakernet`` (This model is trained on voxceleb and telephonic speech Fisher and SWBD. ``Use this model for similar telephonic speech datasets``)
|
||||
- diarizer.speaker_embeddings.window_length_in_sec = 1.5
|
||||
- diarizer.speaker_embeddings.shift_length_in_sec = 0.75
|
||||
|
||||
Input paths2audio_files, paths2rttm_files and oracle_vad_manifest by following steps as shown above
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
@@ -0,0 +1,52 @@
|
||||
Speaker Diarization
|
||||
==================================
|
||||
|
||||
Speaker Diarization (SD) is the task of segmenting audio recordings by speaker labels, that is Who Speaks When?
|
||||
|
||||
.. image:: images/sd_pipeline.png
|
||||
:align: center
|
||||
:alt: Speaker Diarization pipeline [todo]
|
||||
:scale: 50%
|
||||
|
||||
|
||||
A diarization system consists of a **Voice Activity Detection (VAD)** model to get the time stamps of audio where speech is being spoken while ignoring the background noise and
|
||||
a **Speaker Embeddings** model to get speaker embeddings on speech segments obtained from VAD time stamps.
|
||||
These speaker embeddings would then be clustered into clusters based on number of speakers present in the audio recording.
|
||||
|
||||
In NeMo we support both **oracle VAD** and **non-oracle VAD** diarization.
|
||||
|
||||
|
||||
The full documentation tree is as follows:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 8
|
||||
|
||||
models
|
||||
datasets
|
||||
results
|
||||
configs
|
||||
api
|
||||
|
||||
Resource and Documentation Guide
|
||||
--------------------------------
|
||||
|
||||
Hands-on speaker diarization tutorial notebooks can be found under ``<NeMo_git_root>/tutorials/speaker_recognition/``.
|
||||
|
||||
There are tutorials for peformming inference using :ref:`MarbleNet_model` and :ref:`SpeakerNet_model`,
|
||||
and how one can get ASR transcriptions combined with Speaker labels along with voice activity time stamps with NeMo asr collections.
|
||||
|
||||
Most of the tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab.
|
||||
|
||||
If you are looking for information about a particular model used for speaker diarization inference, or would like to find out more about the model
|
||||
architectures available in the `nemo_asr` collection, check out the :doc:`Models <./models>` page.
|
||||
|
||||
Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page.
|
||||
NeMo includes preprocessing scripts for several common ASR datasets, and this page contains instructions on running
|
||||
those scripts.
|
||||
It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data.
|
||||
|
||||
Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list
|
||||
of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page.
|
||||
|
||||
Documentation for configuration files specific to the ``nemo_asr`` models can be found on the
|
||||
:doc:`Configuration Files <./configs>` page.
|
||||
@@ -0,0 +1,5 @@
|
||||
Models
|
||||
======
|
||||
|
||||
|
||||
Currenlty NeMo's Speaker Diarization pipeline uses `MarbleNet <../speech_classification/models.html#marblenet-vad>`__ model for Voice Activity Detection (VAD) and `SpeakerNet <../speaker_recognition/models.html#speakernet>`__ model for Speaker Embedding Extraction.
|
||||
@@ -0,0 +1,76 @@
|
||||
Checkpoints
|
||||
===========
|
||||
|
||||
There are two main ways to load pretrained checkpoints in NeMo as introduced in `loading ASR checkpoints <../results.html#checkpoints>`__.
|
||||
In speaker diarization, the diarizer loads checkpoints that are passed through the config file. For example:
|
||||
|
||||
Loading Local Checkpoints
|
||||
---------------------------
|
||||
|
||||
Load VAD model
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pretrained_vad_model='/path/to/vad_marblenet.nemo' # local .nemo or pretrained vad model name
|
||||
...
|
||||
# pass with hydra config
|
||||
config.diarizer.vad.model_path=pretrained_vad_model
|
||||
|
||||
|
||||
Load Speaker Embedding model
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pretrained_speaker_model='/path/to/speakerdiarization_speakernet.nemo' # local .nemo or pretrained speakernet model name
|
||||
...
|
||||
# pass with hydra config
|
||||
config.diarizer.speaker_embeddings.model_path=pretrained_speaker_model
|
||||
|
||||
|
||||
NeMo will automatically save checkpoints of a model you are training in a `.nemo` format.
|
||||
You can also manually save your models at any point using :code:`model.save_to(<checkpoint_path>.nemo)`.
|
||||
|
||||
|
||||
Inference
|
||||
----------
|
||||
|
||||
.. note::
|
||||
For details and deep understanding, please refer to ``<NeMo_git_root>/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb``.
|
||||
|
||||
Check out :doc:`Datasets <./datasets>` for preparing audio files and optional label files.
|
||||
|
||||
Run and evaluate speaker diarizer with below command:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# Have a look at the instruction inside the script and pass the arguments you might need.
|
||||
python <NeMo_git_root>/examples/speaker_recognition/speaker_diarize.py
|
||||
|
||||
|
||||
NGC Pretrained Checkpoints
|
||||
----------------------------
|
||||
|
||||
The ASR collection has checkpoints of several models trained on various datasets for a variety of tasks.
|
||||
These checkpoints are obtainable via NGC `NeMo Automatic Speech Recognition collection <https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels>`_.
|
||||
The model cards on NGC contain more information about each of the checkpoints available.
|
||||
|
||||
In general, you can load models with model name in the following format,
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
pretrained_vad_model='vad_telephony_marblenet'
|
||||
pretrained_speaker_model='speakerdiarization_speakernet'
|
||||
...
|
||||
config.diarizer.vad.model_path=retrained_vad_model \
|
||||
config.diarizer.speaker_embeddings.model_path=pretrained_speaker_model
|
||||
|
||||
where the model name is the value under "Model Name" entry in the tables below.
|
||||
|
||||
Models for Speaker Diarization
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. csv-table::
|
||||
:file: data/diarization_results.csv
|
||||
:align: left
|
||||
:widths: 30, 30, 40
|
||||
:header-rows: 1
|
||||
@@ -0,0 +1,117 @@
|
||||
NeMo ASR Configuration Files
|
||||
============================
|
||||
|
||||
This page covers NeMo configuration file setup that is specific to speaker recognition models.
|
||||
For general information about how to set up and run experiments that is common to all NeMo models (e.g.
|
||||
experiment manager and PyTorch Lightning trainer parameters), see the :doc:`../../core/core` page.
|
||||
|
||||
The model section of NeMo ASR configuration files will generally require information about the dataset(s) being
|
||||
used, the preprocessor for audio files, parameters for any augmentation being performed, as well as the
|
||||
model architecture specification.
|
||||
The sections on this page cover each of these in more detail.
|
||||
|
||||
Example configuration files for all of the Speaker related scripts can be found in the
|
||||
config directory of the examples ``{NEMO_ROOT/examples/speaker_recognition/conf}``.
|
||||
|
||||
|
||||
Dataset Configuration
|
||||
---------------------
|
||||
|
||||
Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and
|
||||
``test_ds`` sections of your configuration file, respectively.
|
||||
Depending on the task, you may have arguments specifying the sample rate of your audio files, max time length to consider for each audio file , whether or not to shuffle the dataset, and so on.
|
||||
You may also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command line
|
||||
at runtime.
|
||||
|
||||
Any initialization parameters that are accepted for the Dataset class used in your experiment
|
||||
can be set in the config file.
|
||||
|
||||
An example SpeakerNet train and validation configuration could look like:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: None # finds labels based on manifest file
|
||||
batch_size: 32
|
||||
trim_silence: False
|
||||
time_length: 8
|
||||
shuffle: True
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: 16000
|
||||
labels: None # Keep None, to match with labels extracted during training
|
||||
batch_size: 32
|
||||
shuffle: False # No need to shuffle the validation data
|
||||
|
||||
|
||||
If you would like to use tarred dataset, have a look at `Datasets Configuration <../configs.html#dataset-configuration>`__.
|
||||
|
||||
|
||||
Preprocessor Configuration
|
||||
--------------------------
|
||||
Preprocessor helps to compute MFCC or mel spectrogram features that are given as inputs to model.
|
||||
For details on how to write this section, refer to `Preprocessor Configuration <../configs.html#preprocessor-configuration>`__
|
||||
|
||||
|
||||
Augmentation Configurations
|
||||
---------------------------
|
||||
|
||||
For SpeakerNet training we use on-the fly augmentations with MUSAN and RIR impulses using ``noise`` augmentor section
|
||||
|
||||
The following example sets up musan augmentation with audio files taken from manifest path and
|
||||
minimum and maximum SNR specified with min_snr and max_snr respectively. This section can be added to
|
||||
``train_ds`` part in model
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
train_ds:
|
||||
...
|
||||
augmentor:
|
||||
noise:
|
||||
manifest_path: /path/to/musan/manifest_file
|
||||
prob: 0.2 # probability to augment the incoming batch audio with augmentor data
|
||||
min_snr_db: 5
|
||||
max_snr_db: 15
|
||||
|
||||
|
||||
See the :class:`nemo.collections.asr.parts.preprocessing.perturb.AudioAugmentor` API section for more details.
|
||||
|
||||
|
||||
Model Architecture Configurations
|
||||
---------------------------------
|
||||
|
||||
Each configuration file should describe the model architecture being used for the experiment.
|
||||
Models in the NeMo ASR collection need a ``encoder`` section and a ``decoder`` section, with the ``_target_`` field
|
||||
specifying the module to use for each.
|
||||
|
||||
The following sections go into more detail about the specific configurations of each model architecture.
|
||||
|
||||
For more information about the SpeakerNet Encoder models, see the :doc:`Models <./models>` page and at `Jasper and QuartzNet <../configs.html#jasper-and-quartznet>`__
|
||||
|
||||
Decoder Configurations
|
||||
------------------------
|
||||
|
||||
After features have been computed from speakernet encoder, we pass these features to the decoder to compute embeddings and then to compute log probabilities
|
||||
for training models.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.SpeakerDecoder
|
||||
feat_in: *enc_final
|
||||
num_classes: 7205 # Total number of classes in training manifest file
|
||||
pool_mode: xvector # xvector for variance and mean bases statistics pooling
|
||||
emb_sizes: 256 # number of inermediate emb layers. can be comma separated for additional layers like 512,512
|
||||
angular: true # if true then loss will be changed to angular softmax loss and consider scale and margin from loss section else train with cross-entrophy loss
|
||||
|
||||
loss:
|
||||
scale: 30
|
||||
margin 0.2
|
||||
@@ -0,0 +1,4 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
speakerverification_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerverification_speakernet"
|
||||
speakerrecognition_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerrecognition_speakernet"
|
||||
speakerdiarization_speakernet,EncDecSpeakerLabelModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerdiarization_speakernet"
|
||||
|
@@ -0,0 +1,57 @@
|
||||
Datasets
|
||||
========
|
||||
|
||||
.. _HI-MIA:
|
||||
|
||||
HI-MIA
|
||||
--------
|
||||
|
||||
Run the script to download and process ``hi-mia`` dataset in order to generate files in the supported format of ``nemo_asr``. You should set the data folder of
|
||||
hi-mia using ``--data_root``. These scripts are present in ``<nemo_root>/scripts``
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python get_hi-mia_data.py --data_root=<data directory>
|
||||
|
||||
After download and conversion, your `data` folder should contain directories with following set of files as:
|
||||
|
||||
* `data/<set>/train.json`
|
||||
* `data/<set>/dev.json`
|
||||
* `data/<set>/{set}_all.json`
|
||||
* `data/<set>/utt2spk`
|
||||
|
||||
|
||||
All-other Datasets
|
||||
------------------
|
||||
|
||||
These methods can be applied to any dataset to get similar training manifest files.
|
||||
|
||||
First we prepare scp file(s) containing absolute paths to all the wav files required for each of the train, dev, and test set. This can be easily prepared by using ``find`` bash command as follows:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
!find {data_dir}/{train_dir} -iname "*.wav" > data/train_all.scp
|
||||
!head -n 3 data/train_all.scp
|
||||
|
||||
|
||||
Since we created the scp file for the train, we use `scp_to_manifest.py` to convert this scp file to a manifest file and then optionally split the files to train \& dev for evaluating the models while training by using the --split flag.
|
||||
We wont be needing the --split option for the test folder. Accordingly please mention the id number, which is the field num separated by / to be considered as the speaker label.
|
||||
After the download and conversion, your data folder should contain directories with manifest files as:
|
||||
|
||||
* `data/<path>/train.json`
|
||||
* `data/<path>/dev.json`
|
||||
* `data/<path>/train_all.json`
|
||||
|
||||
Each line in the manifest file describes a training sample - audio_filepath contains the path to the wav file, duration it's duration in seconds, and label is the speaker class label:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{"audio_filepath": "<absolute path to dataset>/audio_file.wav", "duration": 3.9, "label": "speaker_id"}
|
||||
|
||||
|
||||
Tarred Datasets
|
||||
---------------
|
||||
|
||||
Similarly to ASR, you can tar your audio files and use ASR Dataset class ``TarredAudioToSpeechLabelDataset`` (corresponding to the ``AudioToSpeechLabelDataset``) for this case.
|
||||
|
||||
If you want to use tarred dataset, have a look at `ASR Tarred Datasets <../datasets.html#tarred-datasets>`__.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 207 KiB |
@@ -0,0 +1,48 @@
|
||||
Speaker Recognition (SR)
|
||||
========================
|
||||
|
||||
Speaker Recognition (SR) is a broad research area which solves two major tasks: speaker identification (who is speaking?) and speaker verification (is the speaker who they claim to be?).
|
||||
We focus on far-field, text-independent speaker recognition when the identity of the speaker is based on how the speech is spoken, not necessarily in what is being said.
|
||||
Typically such SR systems operate on unconstrained speech utterances, which are converted into vectors of fixed length, called speaker embeddings.
|
||||
Speaker embeddings can also be used in automatic speech recognition (ASR) and speech synthesis.
|
||||
|
||||
As the goal of most speaker recognition systems is to get good speaker level embeddings that could help distinguish from other speakers, we shall first train these embeddings in end-to-end manner optimizing the QuatzNet based encoder model on cross-entropy loss.
|
||||
We modify the decoder to get these fixed size embeddings irrespective of the length of the input audio and employ a mean and variance based statistics pooling method to grab these embeddings.
|
||||
|
||||
In Speaker Identification we typically train on a larger training set with cross-entrophy loss and finetune later on your preferred set of labels where one would want to classify only known set of speakers.
|
||||
And in Speaker verification we train with Angular Softmax loss and compare embedings extracted from one audio file coming from a single speaker with
|
||||
embeddings extracted from another file of same or another speaker by employing backend scoring techniques like cosine similarity.
|
||||
|
||||
|
||||
The full documentation tree is as follows:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 8
|
||||
|
||||
models
|
||||
configs
|
||||
datasets
|
||||
results
|
||||
|
||||
Resource and Documentation Guide
|
||||
--------------------------------
|
||||
|
||||
Hands-on speaker recognition tutorial notebooks can be found under
|
||||
`the speaker recognition tutorials folder <https://github.com/NVIDIA/NeMo/tree/v1.0.2/tutorials/speaker_recognition/>`_. This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab.
|
||||
|
||||
If you are looking for information about a particular SpeakerNet model, or would like to find out more about the model
|
||||
architectures available in the ``nemo_asr`` collection, check out the :doc:`Models <./models>` page.
|
||||
|
||||
Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page.
|
||||
NeMo includes preprocessing and other scripts for speaker_recognition in <nemo/scripts/speaker_recognition/> folder, and this page contains instructions on running
|
||||
those scripts. It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data.
|
||||
|
||||
Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list
|
||||
of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page.
|
||||
|
||||
Documentation for configuration files specific to the ``nemo_asr`` models can be found on the
|
||||
:doc:`Configuration Files <./configs>` page.
|
||||
|
||||
|
||||
For a clear step-by-step tutorial we advice you to refer tutorials found in `folder <https://github.com/NVIDIA/NeMo/tree/v1.0.2/tutorials/speaker_recognition/>`_.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
Models
|
||||
======
|
||||
|
||||
Examples of config files for the following quartznet model can be found in the ``<NeMo_git_root>/examples/speaker_recogniton/conf`` directory.
|
||||
|
||||
For more information about the config files and how they should be structured, see the :doc:`./configs` page.
|
||||
|
||||
Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found on the :doc:`./results` page.
|
||||
You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets.
|
||||
The Checkpoints page also contains benchmark results for the available speaker recognition models.
|
||||
|
||||
.. _SpeakerNet_model:
|
||||
|
||||
SpeakerNet
|
||||
-----------
|
||||
|
||||
The model is based on the QuartzNet ASR architecture :cite:`sr-models-koluguri2020speakernet`
|
||||
comprising of an encoder and decoder structure. We use the encoder of the QuartzNet model as a top-level feature extractor, and feed the output to the statistics pooling layer, where
|
||||
we compute the mean and variance across channel dimensions to capture the time-independent utterance-level speaker features.
|
||||
|
||||
The QuartzNet encoder used for speaker embeddings shown in figure below has the following structure: a QuartzNet BxR
|
||||
model has B blocks, each with R sub-blocks. Each sub-block applies the following operations: a 1D convolution, batch norm, ReLU, and dropout. All sub-blocks in a block have the same number of output channels. These blocks are connected with residual connections. We use QuartzNet with 3 blocks, 2 sub-blocks, and 512 channels, as the Encoder for Speaker Embeddings. All conv layers have stride 1 and dilation 1.
|
||||
|
||||
|
||||
.. image:: images/ICASPP_SpeakerNet.png
|
||||
:align: center
|
||||
:alt: speakernet model
|
||||
:scale: 40%
|
||||
|
||||
Top level acoustic Features, obtained from the output of
|
||||
encoder are used to compute intermediate features that are
|
||||
then passed to the decoder for getting utterance level speaker
|
||||
embeddings. The intermediate time-independent features are
|
||||
computed using a statistics pooling layer, where we compute the mean and standard deviation of features across
|
||||
time-channels, to get a time-independent feature representation S of size Batch_size × 3000.
|
||||
The intermediate features, S are passed through the Decoder consisting of two layers each of output size 512 for a
|
||||
linear transformation from S to the final number of classes
|
||||
N for the larger (L) model, and a single linear layer of output size 256 to the final number of classes N for the medium
|
||||
(M) model. We extract q-vectors after the final linear layer
|
||||
of fixed size 512, 256 for SpeakerNet-L and SpeakerNet-M
|
||||
models respectively.
|
||||
|
||||
SpeakerNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecSpeakerLabelModel` class.
|
||||
|
||||
|
||||
References
|
||||
-----------
|
||||
|
||||
.. bibliography:: ../asr_all.bib
|
||||
:style: plain
|
||||
:labelprefix: SR-MODELS
|
||||
:keyprefix: sr-models-
|
||||
@@ -0,0 +1,85 @@
|
||||
Checkpoints
|
||||
===========
|
||||
|
||||
There are two main ways to load pretrained checkpoints in NeMo:
|
||||
|
||||
* Using the :code:`restore_from()` method to load a local checkpoint file (`.nemo`), or
|
||||
* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC.
|
||||
|
||||
See the following sections for instructions and examples for each.
|
||||
|
||||
Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning.
|
||||
For resuming an unfinished training experiment, please use the experiment manager to do so by setting the
|
||||
``resume_if_exists`` flag to True.
|
||||
|
||||
Loading Local Checkpoints
|
||||
-------------------------
|
||||
|
||||
NeMo will automatically save checkpoints of a model you are training in a `.nemo` format.
|
||||
You can also manually save your models at any point using :code:`model.save_to(<checkpoint_path>.nemo)`.
|
||||
|
||||
If you have a local ``.nemo`` checkpoint that you'd like to load, simply use the :code:`restore_from()` method:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.<MODEL_BASE_CLASS>.restore_from(restore_path="<path/to/checkpoint/file.nemo>")
|
||||
|
||||
Where the model base class is the ASR model class of the original checkpoint, or the general `ASRModel` class.
|
||||
|
||||
|
||||
Inference
|
||||
----------
|
||||
|
||||
The audio files should be 16KHz monochannel wav files.
|
||||
|
||||
Write audio files to a ``manifest.json`` file with lines as in format:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{"audio_filepath": "<absolute path to dataset>/audio_file.wav", "duration": "duration of file in sec", "label": "speaker_id"}
|
||||
|
||||
This python call will download best pretrained model from NGC and writes embeddings pickle file to current working directory
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/speaker_recognition/extract_speaker_embeddings.py --manifest=manifest.json
|
||||
|
||||
|
||||
NGC Pretrained Checkpoints
|
||||
--------------------------
|
||||
|
||||
The SpeakerNet-ASR collection has checkpoints of several models trained on various datasets for a variety of tasks.
|
||||
`Speaker_Recognition <https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerrecognition_speakernet>`_ and `Speaker_Verification <https://ngc.nvidia.com/catalog/models/nvidia:nemo:speakerverification_speakernet>`_ model cards on NGC contain more information about each of the checkpoints available.
|
||||
|
||||
The tables below list the SpeakerNet models available from NGC, and the models can be accessed via the
|
||||
:code:`from_pretrained()` method inside the EncDecSpeakerLabelModel Model class.
|
||||
|
||||
.. note:: While loading, remember to use EncDecSpeakerLabelModel for recognition tasks and ExtractSpeakerEmbeddingsModel while extracting embeddings.
|
||||
|
||||
In general, you can load any of these models with code in the following format.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.<MODEL_CLASS_NAME>.from_pretrained(model_name="<MODEL_NAME>")
|
||||
|
||||
Where the model name is the value under "Model Name" entry in the tables below.
|
||||
|
||||
If you would like to programatically list the models available for a particular base class, you can use the
|
||||
:code:`list_available_models()` method.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
nemo_asr.models.<MODEL_BASE_CLASS>.list_available_models()
|
||||
|
||||
|
||||
Speaker Recognition Models
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. csv-table::
|
||||
:file: data/speaker_results.csv
|
||||
:align: left
|
||||
:widths: 30, 30, 40
|
||||
:header-rows: 1
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
NeMo Speech Classification Configuration Files
|
||||
================================================
|
||||
|
||||
This page covers NeMo configuration file setup that is specific to models in the Speech Classification collection.
|
||||
For general information about how to set up and run experiments that is common to all NeMo models (e.g.
|
||||
experiment manager and PyTorch Lightning trainer parameters), see the :doc:`../../core/core` page.
|
||||
|
||||
The model section of NeMo Speech Classification configuration files will generally require information about the dataset(s) being
|
||||
used, the preprocessor for audio files, parameters for any augmentation being performed, as well as the
|
||||
model architecture specification.
|
||||
The sections on this page cover each of these in more detail.
|
||||
|
||||
Example configuration files for all of the NeMo ASR scripts can be found in the
|
||||
``<NeMo_git_root>/examples/asr/conf>``.
|
||||
|
||||
|
||||
Dataset Configuration
|
||||
---------------------
|
||||
|
||||
Training, validation, and test parameters are specified using the ``train_ds``, ``validation_ds``, and
|
||||
``test_ds`` sections of your configuration file, respectively.
|
||||
Depending on the task, you may have arguments specifying the sample rate of your audio files, labels, whether or not to shuffle the dataset, and so on.
|
||||
You may also decide to leave fields such as the ``manifest_filepath`` blank, to be specified via the command line
|
||||
at runtime.
|
||||
|
||||
Any initialization parameters that are accepted for the Dataset class used in your experiment
|
||||
can be set in the config file.
|
||||
See the `Datasets <../api.html#Datasets>`__ section of the API for a list of Datasets and their respective parameters.
|
||||
|
||||
An example Speech Classification train and validation configuration could look like:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
sample_rate: 16000
|
||||
repeat: 2 # number of convolutional sub-blocks within a block, R in <MODEL>_[BxRxC]
|
||||
dropout: 0.0
|
||||
kernel_size_factor: 1.0
|
||||
labels: ['bed', 'bird', 'cat', 'dog', 'down', 'eight', 'five', 'four', 'go', 'happy', 'house', 'left', 'marvin',
|
||||
'nine', 'no', 'off', 'on', 'one', 'right', 'seven', 'sheila', 'six', 'stop', 'three', 'tree', 'two', 'up',
|
||||
'wow', 'yes', 'zero']
|
||||
|
||||
train_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
labels: ${model.labels} # Uses the labels above
|
||||
batch_size: 128
|
||||
shuffle: True
|
||||
|
||||
validation_ds:
|
||||
manifest_filepath: ???
|
||||
sample_rate: ${model.sample_rate}
|
||||
labels: ${model.labels} # Uses the labels above
|
||||
batch_size: 128
|
||||
shuffle: False # No need to shuffle the validation data
|
||||
|
||||
If you would like to use tarred dataset, have a look at `Datasets Configuration <../configs.html#dataset-configuration>`__.
|
||||
|
||||
|
||||
Preprocessor Configuration
|
||||
--------------------------
|
||||
Preprocessor helps to compute MFCC or mel spectrogram features that are given as inputs to model.
|
||||
For details on how to write this section, refer to `Preprocessor Configuration <../configs.html#preprocessor-configuration>`__
|
||||
|
||||
Check config yaml files in ``<NeMo_git_root>/examples/asr/conf`` to find the processors been used by speech classification models.
|
||||
|
||||
|
||||
Augmentation Configurations
|
||||
---------------------------
|
||||
|
||||
There are a few on-the-fly spectrogram augmentation options for NeMo ASR, which can be specified by the
|
||||
configuration file using the ``augmentor`` and ``spec_augment`` section.
|
||||
For details on how to write this section, refer to `Augmentation Configuration <../configs.html#augmentation-configurations>`__
|
||||
|
||||
Check config yaml files in ``<NeMo_git_root>/tutorials/asr/conf`` to find the processors been used by speech classification models.
|
||||
|
||||
|
||||
Model Architecture Configurations
|
||||
---------------------------------
|
||||
|
||||
Each configuration file should describe the model architecture being used for the experiment.
|
||||
Models in the NeMo ASR collection need a ``encoder`` section and a ``decoder`` section, with the ``_target_`` field
|
||||
specifying the module to use for each.
|
||||
|
||||
The following sections go into more detail about the specific configurations of each model architecture.
|
||||
|
||||
The `MatchboxNet <./models.html#matchboxnet-speech-commands>`__ and `MarbleNet <./models.html#marblenet-vad>`__ models are very similar, and they are based on `QuartzNet <../models.html#quartznet>`__ and as such the components in their
|
||||
configs are very similar as well.
|
||||
|
||||
Decoder Configurations
|
||||
------------------------
|
||||
|
||||
After features have been computed from ConvASREncoder, we pass the features to decoder to compute embeddings and then to compute log_probs
|
||||
for training models.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
model:
|
||||
...
|
||||
decoder:
|
||||
_target_: nemo.collections.asr.modules.ConvASRDecoderClassification
|
||||
feat_in: *enc_final_filters
|
||||
return_logits: true # return logits if true, else return softmax output
|
||||
pooling_type: 'avg' # AdaptiveAvgPool1d 'avg' or AdaptiveMaxPool1d 'max'
|
||||
@@ -0,0 +1,9 @@
|
||||
Model Name,Model Base Class,Model Card
|
||||
vad_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_marblenet"
|
||||
vad_telephony_marblenet,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:vad_telephony_marblenet"
|
||||
commandrecognition_en_matchboxnet3x1x64_v1,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v1"
|
||||
commandrecognition_en_matchboxnet3x2x64_v1,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v1"
|
||||
commandrecognition_en_matchboxnet3x1x64_v2,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2"
|
||||
commandrecognition_en_matchboxnet3x2x64_v2,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2"
|
||||
commandrecognition_en_matchboxnet3x1x64_v2_subset_task,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x1x64_v2_subset_task"
|
||||
commandrecognition_en_matchboxnet3x2x64_v2_subset_task,EncDecClassificationModel,"https://ngc.nvidia.com/catalog/models/nvidia:nemo:commandrecognition_en_matchboxnet3x2x64_v2_subset_task"
|
||||
|
@@ -0,0 +1,131 @@
|
||||
Datasets
|
||||
========
|
||||
|
||||
NeMo has scripts to convert several common ASR datasets into the format expected by the `nemo_asr` collection.
|
||||
You can get started with those datasets by following the instructions to run those scripts in the section appropriate to each dataset below.
|
||||
|
||||
If you have your own data and want to preprocess it to use with NeMo ASR models, check out the `Preparing Custom Speech Classification Data`_ section at the bottom of the page.
|
||||
|
||||
.. _Freesound-dataset:
|
||||
|
||||
Freesound
|
||||
-----------
|
||||
|
||||
`Freesound <http://www.freesound.org/>`_ is a website that aims to create a huge open collaborative database of audio snippets, samples, recordings, bleeps.
|
||||
Most audio samples are released under Creative Commons licenses that allow their reuse.
|
||||
Researchers and developers can access Freesound content using the Freesound API to retrieve meaningful sound information such as metadata, analysis files, and the sounds themselves.
|
||||
|
||||
**Instructions**
|
||||
|
||||
Go to ``<NeMo_git_root>/scripts/freesound_download_resample`` and follow the below steps to download and convert freedsound data into a format expected by the `nemo_asr` collection.
|
||||
|
||||
1. We will need some required libraries including freesound, requests, requests_oauthlib, joblib, librosa and sox. If they are not installed, please run `pip install -r freesound_requirements.txt`
|
||||
2. Create an API key for freesound.org at https://freesound.org/help/developers/
|
||||
3. Create a python file called `freesound_private_apikey.py` and add lined `api_key = <your Freesound api key> and client_id = <your Freesound client id>`
|
||||
4. Authorize by run `python freesound_download.py --authorize` and visit the website and paste response code
|
||||
5. Feel free to change any arguments in `download_resample_freesound.sh` such as max_samples and max_filesize
|
||||
6. Run `bash download_resample_freesound.sh <numbers of files you want> <download data directory> <resampled data directory>` . For example:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
bash download_resample_freesound.sh 4000 ./freesound ./freesound_resampled_background
|
||||
|
||||
Note that downloading this dataset may take hours. Change categories in download_resample_freesound.sh to include other (speech) categories audio files.
|
||||
Then, you should have 16khz mono wav files in `<resampled data directory>`.
|
||||
|
||||
|
||||
.. _Google-Speech-Commands-Dataset:
|
||||
|
||||
Google Speech Commands Dataset
|
||||
------------------------------
|
||||
|
||||
Google released two versions of the dataset with the first version containing 65k samples over 30 classes and the second containing 110k samples over 35 classes.
|
||||
We refer to these datasets as `v1` and `v2` respectively.
|
||||
|
||||
Run the script `process_speech_commands_data.py` to process Google Speech Commands dataset in order to generate files in the supported format of `nemo_asr`,
|
||||
which can be found in ``<NeMo_git_root>/scripts/dataset_processing/``.
|
||||
You should set the data folder of Speech Commands using :code:`--data_root` and the version of the dataset using :code:`--data_version` as an int.
|
||||
|
||||
You can further rebalance the train set by randomly oversampling files inside the manifest by passing the `--rebalance` flag.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python process_speech_commands_data.py --data_root=<data directory> --data_version=<1 or 2> {--rebalance}
|
||||
|
||||
|
||||
Then, you should have `train_manifest.json`, `validation_manifest.json` and `test_manifest.json`
|
||||
in the directory `{data_root}/google_speech_recognition_v{1/2}`.
|
||||
|
||||
.. note::
|
||||
You should have at least 4GB or 6GB of disk space available if you use v1 or v2 respectively.
|
||||
Also, it will take some time to download and process, so go grab a coffee.
|
||||
|
||||
Each line is a training example.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
{"audio_filepath": "<absolute path to dataset>/two/8aa35b0c_nohash_0.wav", "duration": 1.0, "label": "two"}
|
||||
{"audio_filepath": "<absolute path to dataset>/two/ec5ab5d5_nohash_2.wav", "duration": 1.0, "label": "two"}
|
||||
|
||||
|
||||
|
||||
Speech Command & Freesound for VAD
|
||||
------------------------------------
|
||||
Speech Command & Freesound (SCF) dataset is used to train MarbleNet in the `paper <https://arxiv.org/pdf/2010.13886.pdf>`_. Here we show how to download and process it.
|
||||
This script assumes that you already have the Freesound dataset, if not, have a look at :ref:`Freesound-dataset`.
|
||||
We will use the open-source :ref:`Google-Speech-Commands-Dataset` (we will use V2 of the dataset for SCF dataset, but require very minor changes to support V1 dataset) as our speech data.
|
||||
|
||||
These scripts below will download the Google Speech Commands v2 dataset and convert speech and background data to a format suitable for use with nemo_asr.
|
||||
|
||||
.. note::
|
||||
You may additionally pass :code:`--test_size` or :code:`--val_size` flag for splitting train val and test data.
|
||||
|
||||
You may additionally pass :code:`--seg_len` flag for indicating the segment length. Default is 0.63s.
|
||||
|
||||
You may additionally pass a :code:`-rebalance_method='fixed|over|under'` at the end of the script to rebalance the class samples in the manifest.
|
||||
|
||||
|
||||
|
||||
* `'fixed'`: Fixed number of sample for each class. Train 5000, val 1000, and test 1000. (Change number in script if you want)
|
||||
* `'over'`: Oversampling rebalance method
|
||||
* `'under'`: Undersampling rebalance method
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
mkdir './google_dataset_v2'
|
||||
python process_vad_data.py --out_dir='./manifest/' --speech_data_root='./google_dataset_v2'--background_data_root=<resampled freesound data directory> --log --rebalance_method='fixed'
|
||||
|
||||
|
||||
After download and conversion, your `manifest` folder should contain a few json manifest files:
|
||||
|
||||
* `(balanced_)background_testing_manifest.json`
|
||||
* `(balanced_)background_training_manifest.json`
|
||||
* `(balanced_)background_validation_manifest.json`
|
||||
* `(balanced_)speech_testing_manifest.json`
|
||||
* `(balanced_)speech_training_manifest.json`
|
||||
* `(balanced_)speech_validation_manifest.json`
|
||||
|
||||
Each line is a training example. `audio_filepath` contains path to the wav file, `duration` is duration in seconds, `offset` is offset in seconds, and `label` is label (class):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
{"audio_filepath": "<absolute path to dataset>/two/8aa35b0c_nohash_0.wav", "duration": 0.63, "label": "speech", "offset": 0.0}
|
||||
{"audio_filepath": "<absolute path to dataset>/Emergency_vehicle/id_58368 simambulance.wav", "duration": 0.63, "label": "background", "offset": 4.0}
|
||||
|
||||
|
||||
|
||||
Preparing Custom Speech Classification Data
|
||||
--------------------------------------------
|
||||
|
||||
Preparing Custom Speech Classification Data is almost identical to `Preparing Custom ASR Data <../datasets.html#preparing-custom-asr-data>`__.
|
||||
|
||||
Instead of :code:`text` entry in manifest, you need :code:`label` to determine class of this sample
|
||||
|
||||
|
||||
Tarred Datasets
|
||||
---------------
|
||||
|
||||
Similarly to ASR, you can tar your audio files and use ASR Dataset class ``TarredAudioToClassificationLabelDataset`` (corresponding to the ``AudioToClassificationLabelDataset``) for this case.
|
||||
|
||||
If you would like to use tarred dataset, have a look at `ASR Tarred Datasets <../datasets.html#tarred-datasets>`__.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 384 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 479 KiB |
@@ -0,0 +1,47 @@
|
||||
Speech Classification
|
||||
==================================
|
||||
Speech Classification refers to a set of tasks or problems of getting a program to automatically classify input utterance or audio segment into categories,
|
||||
such as Speech Command Recognition (multi-class), Voice Activity Detection (binary or multi-class), and Audio Sentiment Classification (typically multi-class), etc.
|
||||
|
||||
**Speech Command Recognition** is the task of classifying an input audio pattern into a discrete set of classes.
|
||||
It is a subset of Automatic Speech Recognition, sometimes referred to as Key Word Spotting, in which a model is constantly analyzing speech patterns to detect certain "command" classes.
|
||||
Upon detection of these commands, a specific action can be taken by the system.
|
||||
It is often the objective of command recognition models to be small and efficient so that they can be deployed onto low-power sensors and remain active for long durations of time.
|
||||
|
||||
|
||||
**Voice Activity Detection (VAD)** also known as speech activity detection or speech detection, is the task of predicting which parts of input audio contain speech versus background noise.
|
||||
It is an essential first step for a variety of speech-based applications including Automatic Speech Recognition.
|
||||
It serves to determine which samples to be sent to the model and when to close the microphone.
|
||||
|
||||
|
||||
The full documentation tree is as follows:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 8
|
||||
|
||||
models
|
||||
datasets
|
||||
results
|
||||
configs
|
||||
|
||||
|
||||
Resource and Documentation Guide
|
||||
--------------------------------
|
||||
|
||||
Hands-on speech classification tutorial notebooks can be found under ``<NeMo_git_repo>/tutorials/asr/``.
|
||||
There are training and offline & online microphone inference tutorials for Speech Command Detection and Voice Activity Detection tasks.
|
||||
This and most other tutorials can be run on Google Colab by specifying the link to the notebooks' GitHub pages on Colab.
|
||||
|
||||
If you are looking for information about a particular Speech Classification model or would like to find out more about the model
|
||||
architectures available in the `nemo_asr` collection, check out the :doc:`Models <./models>` page.
|
||||
|
||||
Documentation on dataset preprocessing can be found on the :doc:`Datasets <./datasets>` page.
|
||||
NeMo includes preprocessing scripts for several common ASR datasets, and this page contains instructions on running
|
||||
those scripts.
|
||||
It also includes guidance for creating your own NeMo-compatible dataset, if you have your own data.
|
||||
|
||||
Information about how to load model checkpoints (either local files or pretrained ones from NGC), perform inference, as well as a list
|
||||
of the checkpoints available on NGC are located on the :doc:`Checkpoints <./results>` page.
|
||||
|
||||
Documentation for configuration files specific to the ``nemo_asr`` models can be found on the
|
||||
:doc:`Configuration Files <./configs>` page.
|
||||
@@ -0,0 +1,71 @@
|
||||
Models
|
||||
======
|
||||
|
||||
This page gives a brief overview of the models that NeMo's Speech Classification collection currently supports.
|
||||
For Speech Classification, we support Speech Command (Keyword) Detection and Voice Activity Detection (VAD).
|
||||
|
||||
Each of these models can be used with the example ASR scripts (in the ``<NeMo_git_root>/examples/asr`` directory) by
|
||||
specifying the model architecture in the config file used.
|
||||
Examples of config files for each model can be found in the ``<NeMo_git_root>/examples/asr/conf`` directory.
|
||||
|
||||
For more information about the config files and how they should be structured, see the :doc:`./configs` page.
|
||||
|
||||
Pretrained checkpoints for all of these models, as well as instructions on how to load them, can be found on the :doc:`./results` page.
|
||||
You can use the available checkpoints for immediate inference, or fine-tune them on your own datasets.
|
||||
The Checkpoints page also contains benchmark results for the available ASR models.
|
||||
|
||||
.. _MatchboxNet_model:
|
||||
|
||||
MatchboxNet (Speech Commands)
|
||||
------------------------------
|
||||
|
||||
MatchboxNet :cite:`sc-models-matchboxnet` is an end-to-end neural network for speech command recognition based on `QuartzNet <../models.html#QuartzNet>`__.
|
||||
|
||||
Similarly to QuartzNet, the MatchboxNet family of models are denoted as MatchBoxNet_[BxRxC] where B is the number of blocks, and R is the number of convolutional sub-blocks within a block, and C is the number of channels. Each sub-block contains a 1-D *separable* convolution, batch normalization, ReLU, and dropout:
|
||||
|
||||
.. image:: images/matchboxnet_vertical.png
|
||||
:align: center
|
||||
:alt: MatchboxNet model
|
||||
:scale: 50%
|
||||
|
||||
It can reach state-of-the art accuracy on the Google Speech Commands dataset while having significantly fewer parameters than similar models.
|
||||
The `_v1` and `_v2` are denoted for models trained on `v1` (30-way classification) and `v2` (35-way classification) datasets;
|
||||
And we use _subset_task to represent (10+2)-way subset (10 specific classes + other remaining classes + silence) classification task.
|
||||
|
||||
MatchboxNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecClassificationModel` class.
|
||||
|
||||
.. note::
|
||||
For model details and deep understanding about Speech Command Detedction training, inference, finetuning and etc.,
|
||||
please refer to ``<NeMo_git_root>/tutorials/asr/03_Speech_Commands.ipynb`` and ``<NeMo_git_root>/tutorials/asr/04_Online_Offline_Speech_Commands_Demo.ipynb``.
|
||||
|
||||
|
||||
|
||||
.. _MarbleNet_model:
|
||||
|
||||
MarbleNet (VAD)
|
||||
------------------
|
||||
|
||||
MarbleNet :cite:`sc-models-marblenet` an end-to-end neural network for speech command recognition based on :ref:`MatchboxNet_model`,
|
||||
|
||||
Similarly to MatchboxNet, the MarbleNet family of models are denoted as MarbleNet_[BxRxC] where B is the number of blocks, and R is the number of convolutional sub-blocks within a block, and C is the number of channels. Each sub-block contains a 1-D *separable* convolution, batch normalization, ReLU, and dropout:
|
||||
|
||||
.. image:: images/marblenet_vertical.png
|
||||
:align: center
|
||||
:alt: MarbleNet model
|
||||
:scale: 30%
|
||||
|
||||
It can reach state-of-the art performance on the difficult `AVA speech dataset <https://research.google.com/ava/download.html#ava_speech_download>`_ while having significantly fewer parameters than similar models even training on simple data.
|
||||
MarbleNet models can be instantiated using the :class:`~nemo.collections.asr.models.EncDecClassificationModel` class.
|
||||
|
||||
.. note::
|
||||
For model details and deep understanding about VAD training, inference, postprocessing, threshold tuning and etc.,
|
||||
please refer to ``<NeMo_git_root>/tutorials/asr/06_Voice_Activiy_Detection.ipynb`` and ``<NeMo_git_root>/tutorials/asr/07_Online_Offline_Microphone_VAD_Demo.ipynb``.
|
||||
|
||||
|
||||
References
|
||||
----------------
|
||||
|
||||
.. bibliography:: ../asr_all.bib
|
||||
:style: plain
|
||||
:labelprefix: SC-MODELS
|
||||
:keyprefix: sc-models-
|
||||
@@ -0,0 +1,105 @@
|
||||
Checkpoints
|
||||
===========
|
||||
|
||||
There are two main ways to load pretrained checkpoints in NeMo:
|
||||
|
||||
* Using the :code:`restore_from()` method to load a local checkpoint file (`.nemo`), or
|
||||
* Using the :code:`from_pretrained()` method to download and set up a checkpoint from NGC.
|
||||
|
||||
See the following sections for instructions and examples for each.
|
||||
|
||||
Note that these instructions are for loading fully trained checkpoints for evaluation or fine-tuning.
|
||||
For resuming an unfinished training experiment, please use the experiment manager to do so by setting the
|
||||
``resume_if_exists`` flag to True.
|
||||
|
||||
Loading Local Checkpoints
|
||||
-------------------------
|
||||
|
||||
NeMo will automatically save checkpoints of a model you are training in a `.nemo` format.
|
||||
You can also manually save your models at any point using :code:`model.save_to(<checkpoint_path>.nemo)`.
|
||||
|
||||
If you have a local ``.nemo`` checkpoint that you'd like to load, simply use the :code:`restore_from()` method:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.<MODEL_BASE_CLASS>.restore_from(restore_path="<path/to/checkpoint/file.nemo>")
|
||||
|
||||
Where the model base class is the ASR model class of the original checkpoint, or the general `ASRModel` class.
|
||||
|
||||
|
||||
Transcribing/Inference
|
||||
-----------------------
|
||||
|
||||
The audio files should be 16KHz monochannel wav files.
|
||||
|
||||
**Transcribe speech command segment:**
|
||||
|
||||
You may perform inference and transcribe a sample of speech after loading the model by using its 'transcribe()' method:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
mbn_model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="<MODEL_NAME>")
|
||||
mbn_model.transcribe([list of audio files], batch_size=BATCH_SIZE, logprobs=False)
|
||||
|
||||
Setting argument ``logprobs`` to True would return the log probabilities instead of transcriptions. You may find more details in `Modules <../api.html#modules>`__.
|
||||
|
||||
Learn how to fine tune on your own data or on subset classes in ``<NeMo_git_root>/tutorials/asr/03_Speech_Commands.ipynb``
|
||||
f
|
||||
|
||||
**Run VAD inference:**
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/speaker_recognition/vad_infer.py --vad_model="vad_marblenet" --dataset=<FULL PATH OF MANIFEST TO BE PERFORMED INFERENCE ON> --out_dir='frame/demo' --time_length=0.63
|
||||
|
||||
Have a look at scripts under ``<NeMo-git-root>/scripts/voice_activity_detection`` for posterior processing and threshold tuning.
|
||||
|
||||
|
||||
NGC Pretrained Checkpoints
|
||||
--------------------------
|
||||
|
||||
The Speech Classification collection has checkpoints of several models trained on various datasets for a variety of tasks.
|
||||
These checkpoints are obtainable via NGC `NeMo Automatic Speech Recognition collection <https://ngc.nvidia.com/catalog/models/nvidia:nemospeechmodels>`_.
|
||||
The model cards on NGC contain more information about each of the checkpoints available.
|
||||
|
||||
The tables below list the Speech Classification models available from NGC, and the models can be accessed via the
|
||||
:code:`from_pretrained()` method inside the ASR Model class.
|
||||
|
||||
In general, you can load any of these models with code in the following format.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="<MODEL_NAME>")
|
||||
|
||||
Where the model name is the value under "Model Name" entry in the tables below.
|
||||
|
||||
For example, to load the MatchboxNet3x2x64_v1 model for speech command detection, run:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model = nemo_asr.models.EncDecClassificationModel.from_pretrained(model_name="commandrecognition_en_matchboxnet3x2x64_v1")
|
||||
|
||||
You can also call :code:`from_pretrained()` from the specific model class (such as :code:`EncDecClassificationModel`
|
||||
for MatchboxNet and MarbleNet) if you will need to access specific model functionality.
|
||||
|
||||
If you would like to programatically list the models available for a particular base class, you can use the
|
||||
:code:`list_available_models()` method.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
nemo_asr.models.<MODEL_BASE_CLASS>.list_available_models()
|
||||
|
||||
|
||||
Speech Classification Models
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. tabularcolumns:: 30 30 40
|
||||
|
||||
.. csv-table::
|
||||
:file: data/classification_results.csv
|
||||
:header-rows: 1
|
||||
:class: longtable
|
||||
:widths: 1 1 1
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
Common Collection
|
||||
=================
|
||||
|
||||
The common collection contains things that could be used across all collections.
|
||||
|
||||
Tokenizers
|
||||
----------
|
||||
.. automodule:: nemo.collections.common.tokenizers.AutoTokenizer
|
||||
:special-members: __init__
|
||||
.. automodule:: nemo.collections.common.tokenizers.SentencePieceTokenizer
|
||||
:special-members: __init__
|
||||
.. automodule:: nemo.collections.common.tokenizers.TokenizerSpec
|
||||
:special-members: __init__
|
||||
|
||||
|
||||
Losses
|
||||
------
|
||||
.. automodule:: nemo.collections.common.losses.AggregatorLoss
|
||||
:special-members: __init__
|
||||
|
||||
.. automodule:: nemo.collections.common.losses.CrossEntropyLoss
|
||||
:special-members: __init__
|
||||
|
||||
.. automodule:: nemo.collections.common.losses.MSELoss
|
||||
:special-members: __init__
|
||||
|
||||
.. automodule:: nemo.collections.common.losses.SmoothedCrossEntropyLoss
|
||||
:special-members: __init__
|
||||
.. automodule:: nemo.collections.common.losses.SpanningLoss
|
||||
:special-members: __init__
|
||||
|
||||
|
||||
Metrics
|
||||
-------
|
||||
|
||||
.. autoclass:: nemo.collections.common.metrics.Perplexity
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:undoc-members:
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import glob
|
||||
|
||||
import sphinx_rtd_theme
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
sys.path.insert(0, os.path.abspath("../../nemo"))
|
||||
sys.path.insert(0, os.path.abspath("../../nemo_text_processing"))
|
||||
|
||||
from package_info import __version__
|
||||
|
||||
|
||||
autodoc_mock_imports = [
|
||||
'torch',
|
||||
'torch.nn',
|
||||
'torch.utils',
|
||||
'torch.optim',
|
||||
'torch.utils.data',
|
||||
'torch.utils.data.sampler',
|
||||
'torchvision',
|
||||
'torchvision.models',
|
||||
'torchtext',
|
||||
'ruamel.yaml', # ruamel.yaml has ., which is troublesome for this regex
|
||||
'hydra', # hydra-core in requirements, hydra during import
|
||||
'dateutil', # part of core python
|
||||
'transformers.tokenization_bert', # has ., troublesome for this regex
|
||||
'megatron', # megatron-lm in requirements, megatron in import
|
||||
'sklearn',
|
||||
'nemo_text_processing.inverse_text_normalization', # Not installed automatically
|
||||
'nemo_text_processing.text_normalization', # Not installed automatically
|
||||
'attr', # attrdict in requirements, attr in import
|
||||
'torchmetrics', # inherited from PTL
|
||||
]
|
||||
|
||||
_skipped_autodoc_mock_imports = ['wrapt', 'numpy']
|
||||
|
||||
for req_path in sorted(list(glob.glob("../../requirements/*.txt"))):
|
||||
if "docs.txt" in req_path:
|
||||
continue
|
||||
|
||||
req_file = os.path.abspath(os.path.expanduser(req_path))
|
||||
with open(req_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.replace("\n", "")
|
||||
req = re.search(r"([a-zA-Z0-9-_]*)", line)
|
||||
if req:
|
||||
req = req.group(1)
|
||||
req = req.replace("-", "_")
|
||||
|
||||
if req not in autodoc_mock_imports:
|
||||
if req in _skipped_autodoc_mock_imports:
|
||||
print(f"Skipping req : `{req}` (lib {line})")
|
||||
continue
|
||||
|
||||
autodoc_mock_imports.append(req)
|
||||
print(f"Adding req : `{req}` to autodoc mock requirements (lib {line})")
|
||||
else:
|
||||
print(f"`{req}` already added to autodoc mock requirements (lib {line})")
|
||||
|
||||
#
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
||||
# ones.
|
||||
|
||||
extensions = [
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.todo",
|
||||
"sphinx.ext.coverage",
|
||||
"sphinx.ext.mathjax",
|
||||
"sphinx.ext.ifconfig",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.githubpages",
|
||||
"sphinxcontrib.bibtex",
|
||||
"sphinx.ext.inheritance_diagram",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx.ext.autosectionlabel",
|
||||
"sphinxcontrib.bibtex",
|
||||
"sphinx_rtd_theme",
|
||||
]
|
||||
|
||||
bibtex_bibfiles = [
|
||||
'asr/asr_all.bib',
|
||||
'nlp/nlp_all.bib',
|
||||
'tools/tools_all.bib',
|
||||
'nemo_text_processing/textprocessing_all.bib',
|
||||
'tts_all.bib',
|
||||
]
|
||||
|
||||
intersphinx_mapping = {
|
||||
'pytorch': ('https://pytorch.org/docs/stable', None),
|
||||
'pytorch-lightning': ('https://pytorch-lightning.readthedocs.io/en/latest/', None),
|
||||
}
|
||||
|
||||
# Set default flags for all classes.
|
||||
autodoc_default_options = {'members': None, 'undoc-members': None, 'show-inheritance': True}
|
||||
|
||||
locale_dirs = ['locale/'] # path is example but recommended.
|
||||
gettext_compact = False # optional.
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = ".rst"
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = "NVIDIA NeMo"
|
||||
copyright = "2021-, NVIDIA CORPORATION"
|
||||
author = "NVIDIA CORPORATION"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
|
||||
|
||||
# The short X.Y version.
|
||||
# version = "0.10.0"
|
||||
version = __version__
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
# release = "0.9.0"
|
||||
release = __version__
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = None
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = []
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = "default"
|
||||
|
||||
### Previous NeMo theme
|
||||
# # NVIDIA theme settings.
|
||||
# html_theme = 'nvidia_theme'
|
||||
|
||||
# html_theme_path = ["."]
|
||||
|
||||
# html_theme_options = {
|
||||
# 'display_version': True,
|
||||
# 'project_version': version,
|
||||
# 'project_name': project,
|
||||
# 'logo_path': None,
|
||||
# 'logo_only': True,
|
||||
# }
|
||||
# html_title = 'Introduction'
|
||||
|
||||
# html_logo = html_theme_options["logo_path"]
|
||||
|
||||
# -- Options for HTMLHelp output ------------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = "nemodoc"
|
||||
|
||||
### from TLT conf.py
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
|
||||
# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
|
||||
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
html_logo = os.path.join('nv_logo.png')
|
||||
|
||||
html_theme_options = {
|
||||
'logo_only': True,
|
||||
'display_version': True,
|
||||
'prev_next_buttons_location': 'bottom',
|
||||
'style_external_links': False,
|
||||
'style_nav_header_background': '#000000',
|
||||
# Toc options
|
||||
'collapse_navigation': False,
|
||||
'sticky_navigation': False,
|
||||
# 'navigation_depth': 10,
|
||||
'includehidden': False,
|
||||
'titles_only': False,
|
||||
}
|
||||
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
|
||||
html_favicon = 'favicon.ico'
|
||||
|
||||
html_static_path = ['_static']
|
||||
|
||||
html_last_updated_fmt = ''
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.add_css_file('css/custom.css')
|
||||
app.add_js_file('js/pk_scripts.js')
|
||||
|
||||
|
||||
# html_css_files = [
|
||||
# './custom.css',
|
||||
# ]
|
||||
|
||||
# html_js_files = [
|
||||
# './pk_scripts.js',
|
||||
# ]
|
||||
@@ -0,0 +1,963 @@
|
||||
NeMo Models
|
||||
===========
|
||||
|
||||
Basics
|
||||
------
|
||||
|
||||
NeMo models contain everything needed to train and reproduce Conversational AI models:
|
||||
|
||||
- neural network architectures
|
||||
- datasets/data loaders
|
||||
- data preprocessing/postprocessing
|
||||
- data augmentors
|
||||
- optimizers and schedulers
|
||||
- tokenizers
|
||||
- language models
|
||||
|
||||
NeMo uses `Hydra <https://hydra.cc/>`_ for configuring both NeMo models and the PyTorch Lightning Trainer.
|
||||
|
||||
.. note:: Every NeMo model has an example configuration file and training script that can be found `here <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples>`_.
|
||||
|
||||
The end result of using NeMo, `Pytorch Lightning <https://github.com/PyTorchLightning/pytorch-lightning>`_, and Hydra is that NeMo models all have the same look and feel and are also fully compatible with the PyTorch ecosystem.
|
||||
|
||||
Pretrained
|
||||
----------
|
||||
|
||||
NeMo comes with many pretrained models for each of our collections: ASR, NLP, and TTS. Every pretrained NeMo model can be downloaded
|
||||
and used with the ``from_pretrained()`` method.
|
||||
|
||||
As an example, we can instantiate QuartzNet with the following:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
import nemo.collections.asr as nemo_asr
|
||||
|
||||
model = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="QuartzNet15x5Base-En")
|
||||
|
||||
To see all available pretrained models for a specific NeMo model, use the ``list_available_models()`` method.
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
nemo_asr.model.EncDecCTCModel.list_available_models()
|
||||
|
||||
For detailed information on the available pretrained models, refer to the collections documentation:
|
||||
|
||||
- :ref:`Automatic Speech Recognition (ASR)`
|
||||
- :ref:`Natural Language Processing (NLP)`
|
||||
- :ref:`Speech Synthesis (TTS)`
|
||||
|
||||
Training
|
||||
--------
|
||||
|
||||
NeMo leverages `PyTorch Lightning <https://www.pytorchlightning.ai/>`_ for model training. PyTorch Lightning lets NeMo decouple the
|
||||
conversational AI code from the PyTorch training code. This means that NeMo users can focus on their domain (ASR, NLP, TTS) and
|
||||
build complex AI applications without having to rewrite boiler plate code for PyTorch training.
|
||||
|
||||
When using PyTorch Lightning, NeMo users can automatically train with:
|
||||
|
||||
- multi-GPU/multi-node
|
||||
- mixed precision
|
||||
- model checkpointing
|
||||
- logging
|
||||
- early stopping
|
||||
- and more
|
||||
|
||||
The two main aspects of the Lightning API are the `LightningModule <https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#>`_
|
||||
and the `Trainer <https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html>`_.
|
||||
|
||||
PyTorch Lightning ``LightningModule``
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Every NeMo model is a ``LightningModule`` which is an ``nn.module``. This means that NeMo models are compatible with the PyTorch
|
||||
ecosystem and can be plugged into existing PyTorch workflows.
|
||||
|
||||
Creating a NeMo model is similar to any other PyTorch workflow. We start by initializing our model architecture, then define the forward pass:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class TextClassificationModel(NLPModel, Exportable):
|
||||
...
|
||||
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
|
||||
"""Initializes the BERTTextClassifier model."""
|
||||
...
|
||||
super().__init__(cfg=cfg, trainer=trainer)
|
||||
|
||||
# instantiate a BERT based encoder
|
||||
self.bert_model = get_lm_model(
|
||||
pretrained_model_name=cfg.language_model.pretrained_model_name,
|
||||
config_file=cfg.language_model.config_file,
|
||||
config_dict=cfg.language_model.config,
|
||||
checkpoint_file=cfg.language_model.lm_checkpoint,
|
||||
vocab_file=cfg.tokenizer.vocab_file,
|
||||
)
|
||||
|
||||
# instantiate the FFN for classification
|
||||
self.classifier = SequenceClassifier(
|
||||
hidden_size=self.bert_model.config.hidden_size,
|
||||
num_classes=cfg.dataset.num_classes,
|
||||
num_layers=cfg.classifier_head.num_output_layers,
|
||||
activation='relu',
|
||||
log_softmax=False,
|
||||
dropout=cfg.classifier_head.fc_dropout,
|
||||
use_transformer_init=True,
|
||||
idx_conditioned_on=0,
|
||||
)
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def forward(self, input_ids, token_type_ids, attention_mask):
|
||||
"""
|
||||
No special modification required for Lightning, define it as you normally would
|
||||
in the `nn.Module` in vanilla PyTorch.
|
||||
"""
|
||||
hidden_states = self.bert_model(
|
||||
input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask
|
||||
)
|
||||
logits = self.classifier(hidden_states=hidden_states)
|
||||
return logits
|
||||
|
||||
The ``LightningModule`` organizes PyTorch code so that across all NeMo models we have a similar look and feel.
|
||||
For example, the training logic can be found in ``training_step``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def training_step(self, batch, batch_idx):
|
||||
"""
|
||||
Lightning calls this inside the training loop with the data from the training dataloader
|
||||
passed in as `batch`.
|
||||
"""
|
||||
# forward pass
|
||||
input_ids, input_type_ids, input_mask, labels = batch
|
||||
logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask)
|
||||
|
||||
train_loss = self.loss(logits=logits, labels=labels)
|
||||
|
||||
lr = self._optimizer.param_groups[0]['lr']
|
||||
|
||||
self.log('train_loss', train_loss)
|
||||
self.log('lr', lr, prog_bar=True)
|
||||
|
||||
return {
|
||||
'loss': train_loss,
|
||||
'lr': lr,
|
||||
}
|
||||
|
||||
While validation logic can be found in ``validation_step``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def validation_step(self, batch, batch_idx):
|
||||
"""
|
||||
Lightning calls this inside the validation loop with the data from the validation dataloader
|
||||
passed in as `batch`.
|
||||
"""
|
||||
if self.testing:
|
||||
prefix = 'test'
|
||||
else:
|
||||
prefix = 'val'
|
||||
|
||||
input_ids, input_type_ids, input_mask, labels = batch
|
||||
logits = self.forward(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask)
|
||||
|
||||
val_loss = self.loss(logits=logits, labels=labels)
|
||||
|
||||
preds = torch.argmax(logits, axis=-1)
|
||||
|
||||
tp, fn, fp, _ = self.classification_report(preds, labels)
|
||||
|
||||
return {'val_loss': val_loss, 'tp': tp, 'fn': fn, 'fp': fp}
|
||||
|
||||
PyTorch Lightning then handles all of the boiler plate code needed for training. Virtually any aspect of training can be customized
|
||||
via PyTorch Lightning `hooks <https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#hooks>`_,
|
||||
`Plugins <https://pytorch-lightning.readthedocs.io/en/stable/extensions/plugins.html>`_,
|
||||
`callbacks <https://pytorch-lightning.readthedocs.io/en/stable/extensions/callbacks.html>`_, or by overriding `methods <https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#methods>`_.
|
||||
|
||||
For more domain-specific information, see:
|
||||
|
||||
- :ref:`Automatic Speech Recognition (ASR)`
|
||||
- :ref:`Natural Language Processing (NLP)`
|
||||
- :ref:`Speech Synthesis (TTS)`
|
||||
|
||||
PyTorch Lightning Trainer
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Since every NeMo model is a ``LightningModule``, we can automatically take advantage of the PyTorch Lightning ``Trainer``. Every NeMo
|
||||
`example <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples>`_ training script uses the ``Trainer`` object to fit the model.
|
||||
|
||||
First, instantiate the model and trainer, then call ``.fit``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# We first instantiate the trainer based on the model configuration.
|
||||
# See the model configuration documentation for details.
|
||||
trainer = pl.Trainer(**cfg.trainer)
|
||||
|
||||
# Then pass the model configuration and trainer object into the NeMo model
|
||||
model = TextClassificationModel(cfg.model, trainer=trainer)
|
||||
|
||||
# Now we can train with by calling .fit
|
||||
trainer.fit(model)
|
||||
|
||||
# Or we can run the test loop on test data by calling
|
||||
trainer.test(model=model)
|
||||
|
||||
All `trainer flags <https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#trainer-flags>`_ can be set from from the
|
||||
NeMo configuration.
|
||||
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
|
||||
Hydra is an open-source Python framework that simplifies configuration for complex applications that must bring together many different
|
||||
software libraries. Conversational AI model training is a great example of such an application. To train a conversational AI model, we
|
||||
must be able to configure:
|
||||
|
||||
- neural network architectures
|
||||
- training and optimization algorithms
|
||||
- data pre/post processing
|
||||
- data augmentation
|
||||
- experiment logging/visualization
|
||||
- model checkpointing
|
||||
|
||||
For an introduction to using Hydra, refer to the `Hydra Tutorials <https://hydra.cc/docs/tutorials/intro>`_.
|
||||
|
||||
With Hydra, we can configure everything needed for NeMo with three interfaces:
|
||||
|
||||
- Command Line (CLI)
|
||||
- Configuration Files (YAML)
|
||||
- Dataclasses (Python)
|
||||
|
||||
YAML
|
||||
~~~~
|
||||
|
||||
NeMo provides YAML configuration files for all of our `example <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples>`_ training scripts.
|
||||
YAML files make it easy to experiment with different model and training configurations.
|
||||
|
||||
Every NeMo example YAML has the same underlying configuration structure:
|
||||
|
||||
- trainer
|
||||
- exp_manager
|
||||
- model
|
||||
|
||||
Model configuration always contain ``train_ds``, ``validation_ds``, ``test_ds``, and ``optim``. Model architectures vary across
|
||||
domains, therefore, refer to the ASR, NLP, and TTS Collections documentation for more detailed information on Model architecture configuration.
|
||||
|
||||
A NeMo configuration file should look similar to the following:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
# PyTorch Lightning Trainer configuration
|
||||
# any argument of the Trainer object can be set here
|
||||
trainer:
|
||||
gpus: 1 # number of gpus per node
|
||||
num_nodes: 1 # number of nodes
|
||||
max_epochs: 10 # how many training epochs to run
|
||||
val_check_interval: 1.0 # run validation after every epoch
|
||||
|
||||
# Experiment logging configuration
|
||||
exp_manager:
|
||||
exp_dir: /path/to/my/nemo/experiments
|
||||
name: name_of_my_experiment
|
||||
create_tensorboard_logger: True
|
||||
create_wandb_logger: True
|
||||
|
||||
# Model configuration
|
||||
# model network architecture, train/val/test datasets, data augmentation, and optimization
|
||||
model:
|
||||
train_ds:
|
||||
manifest_filepath: /path/to/my/train/manifest.json
|
||||
batch_size: 256
|
||||
shuffle: True
|
||||
validation_ds:
|
||||
manifest_filepath: /path/to/my/validation/manifest.json
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
test_ds:
|
||||
manifest_filepath: /path/to/my/test/manifest.json
|
||||
batch_size: 32
|
||||
shuffle: False
|
||||
optim:
|
||||
name: novograd
|
||||
lr: .01
|
||||
betas: [0.8, 0.5]
|
||||
weight_decay: 0.001
|
||||
# network architecture can vary greatly depending on the domain
|
||||
encoder:
|
||||
...
|
||||
decoder:
|
||||
...
|
||||
|
||||
More specific details about configuration files for each collection can be found on the following pages:
|
||||
|
||||
:ref:`NeMo ASR Configuration Files`
|
||||
|
||||
CLI
|
||||
~~~
|
||||
|
||||
With NeMo and Hydra, every aspect of model training can be modified from the command-line. This is extremely helpful for running lots
|
||||
of experiments on compute clusters or for quickly testing parameters while developing.
|
||||
|
||||
All NeMo `examples <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples>`_ come with instructions on how to
|
||||
run the training/inference script from the command-line (see `here <https://github.com/NVIDIA/NeMo/blob/4e9da75f021fe23c9f49404cd2e7da4597cb5879/examples/asr/speech_to_text.py#L24>`_
|
||||
for an example).
|
||||
|
||||
With Hydra, arguments are set using the ``=`` operator:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/asr/speech_to_text.py \
|
||||
model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \
|
||||
model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=50
|
||||
|
||||
We can use the ``+`` operator to add arguments from the CLI:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/asr/speech_to_text.py \
|
||||
model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \
|
||||
model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=50 \
|
||||
+trainer.fast_dev_run=true
|
||||
|
||||
We can use the ``~`` operator to remove configurations:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/asr/speech_to_text.py \
|
||||
model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \
|
||||
model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \
|
||||
~model.test_ds \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=50 \
|
||||
+trainer.fast_dev_run=true
|
||||
|
||||
We can specify configuration files using the ``--config-path`` and ``--config-name`` flags:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/asr/speech_to_text.py \
|
||||
--config-path=conf/quartznet \
|
||||
--config-name=quartznet_15x5 \
|
||||
model.train_ds.manifest_filepath=/path/to/my/train/manifest.json \
|
||||
model.validation_ds.manifest_filepath=/path/to/my/validation/manifest.json \
|
||||
~model.test_ds \
|
||||
trainer.gpus=2 \
|
||||
trainer.max_epochs=50 \
|
||||
+trainer.fast_dev_run=true
|
||||
|
||||
Dataclasses
|
||||
~~~~~~~~~~~
|
||||
|
||||
Dataclasses allow NeMo to ship model configurations as part of the NeMo library and also enables pure Python configuration of NeMo models.
|
||||
With Hydra, dataclasses can be used to create `structured configs <https://hydra.cc/docs/tutorials/structured_config/intro>`_ for the conversational AI application.
|
||||
|
||||
As an example, refer to the code block below for an *Attenion is All You Need* machine translation model. The model configuration can
|
||||
be instantiated and modified like any Python `Dataclass <https://docs.python.org/3/library/dataclasses.html>`_.
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
from nemo.collections.nlp.models.machine_translation.mt_enc_dec_config import AAYNBaseConfig
|
||||
|
||||
cfg = AAYNBaseConfig()
|
||||
|
||||
# modify the number of layers in the encoder
|
||||
cfg.encoder.num_layers = 8
|
||||
|
||||
# modify the training batch size
|
||||
cfg.train_ds.tokens_in_batch = 8192
|
||||
|
||||
.. note:: Configuration with Hydra always has the following precedence CLI > YAML > Dataclass
|
||||
|
||||
Optimization
|
||||
------------
|
||||
|
||||
Optimizers and learning rate schedules are configurable across all NeMo models and have their own namespace. Here is a sample YAML
|
||||
configuration for a Novograd optimizer with Cosine Annealing learning rate schedule.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
optim:
|
||||
name: novograd
|
||||
lr: 0.01
|
||||
|
||||
# optimizer arguments
|
||||
betas: [0.8, 0.25]
|
||||
weight_decay: 0.001
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
|
||||
# Optional arguments
|
||||
max_steps: null # computed at runtime or explicitly set here
|
||||
monitor: val_loss
|
||||
reduce_on_plateau: false
|
||||
|
||||
# scheduler config override
|
||||
warmup_steps: 1000
|
||||
warmup_ratio: null
|
||||
min_lr: 1e-9:
|
||||
|
||||
.. note:: `NeMo Examples <https://github.com/NVIDIA/NeMo/tree/v1.0.2/examples>`_ has optimizer and scheduler configurations for
|
||||
every NeMo model.
|
||||
|
||||
Optimizers can be configured from the CLI as well:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python examples/asr/speech_to_text.py \
|
||||
--config-path=conf/quartznet \
|
||||
--config-name=quartznet_15x5 \
|
||||
...
|
||||
# train with the adam optimizer
|
||||
model.optim=adam \
|
||||
# change the learning rate
|
||||
model.optim.lr=.0004 \
|
||||
# modify betas
|
||||
model.optim.betas=[.8, .5]
|
||||
|
||||
Optimizers
|
||||
~~~~~~~~~~
|
||||
|
||||
``name`` corresponds to the lowercase name of the optimizer. To view a list of available optimizers, run:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
from nemo.core.optim.optimizers import AVAILABLE_OPTIMIZERS
|
||||
|
||||
for name, opt in AVAILABLE_OPTIMIZERS.items():
|
||||
print(f'name: {name}, opt: {opt}')
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
name: sgd opt: <class 'torch.optim.sgd.SGD'>
|
||||
name: adam opt: <class 'torch.optim.adam.Adam'>
|
||||
name: adamw opt: <class 'torch.optim.adamw.AdamW'>
|
||||
name: adadelta opt: <class 'torch.optim.adadelta.Adadelta'>
|
||||
name: adamax opt: <class 'torch.optim.adamax.Adamax'>
|
||||
name: adagrad opt: <class 'torch.optim.adagrad.Adagrad'>
|
||||
name: rmsprop opt: <class 'torch.optim.rmsprop.RMSprop'>
|
||||
name: rprop opt: <class 'torch.optim.rprop.Rprop'>
|
||||
name: novograd opt: <class 'nemo.core.optim.novograd.Novograd'>
|
||||
|
||||
Optimizer Params
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Optimizer params can vary between optimizers but the ``lr`` param is required for all optimizers. To see the available params for an
|
||||
optimizer, we can look at its corresponding dataclass.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.core.config.optimizers import NovogradParams
|
||||
|
||||
print(NovogradParams())
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
NovogradParams(lr='???', betas=(0.95, 0.98), eps=1e-08, weight_decay=0, grad_averaging=False, amsgrad=False, luc=False, luc_trust=0.001, luc_eps=1e-08)
|
||||
|
||||
``'???'`` indicates that the lr argument is required.
|
||||
|
||||
Register Optimizer
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To register a new optimizer to be used with NeMo, run:
|
||||
|
||||
.. autofunction:: nemo.core.optim.optimizers.register_optimizer
|
||||
|
||||
Learning Rate Schedulers
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Learning rate schedulers can be optionally configured under the ``optim.sched`` namespace.
|
||||
|
||||
``name`` corresponds to the name of the learning rate schedule. To view a list of available schedulers, run:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
from nemo.core.optim.lr_scheduler import AVAILABLE_SCHEDULERS
|
||||
|
||||
for name, opt in AVAILABLE_SCHEDULERS.items():
|
||||
print(f'name: {name}, schedule: {opt}')
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
name: WarmupPolicy, schedule: <class 'nemo.core.optim.lr_scheduler.WarmupPolicy'>
|
||||
name: WarmupHoldPolicy, schedule: <class 'nemo.core.optim.lr_scheduler.WarmupHoldPolicy'>
|
||||
name: SquareAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.SquareAnnealing'>
|
||||
name: CosineAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.CosineAnnealing'>
|
||||
name: NoamAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.NoamAnnealing'>
|
||||
name: WarmupAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.WarmupAnnealing'>
|
||||
name: InverseSquareRootAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing'>
|
||||
name: SquareRootAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.SquareRootAnnealing'>
|
||||
name: PolynomialDecayAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.PolynomialDecayAnnealing'>
|
||||
name: PolynomialHoldDecayAnnealing, schedule: <class 'nemo.core.optim.lr_scheduler.PolynomialHoldDecayAnnealing'>
|
||||
name: StepLR, schedule: <class 'torch.optim.lr_scheduler.StepLR'>
|
||||
name: ExponentialLR, schedule: <class 'torch.optim.lr_scheduler.ExponentialLR'>
|
||||
name: ReduceLROnPlateau, schedule: <class 'torch.optim.lr_scheduler.ReduceLROnPlateau'>
|
||||
name: CyclicLR, schedule: <class 'torch.optim.lr_scheduler.CyclicLR'>
|
||||
|
||||
Scheduler Params
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
To see the available params for a scheduler, we can look at its corresponding dataclass:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
from nemo.core.config.schedulers import CosineAnnealingParams
|
||||
|
||||
print(CosineAnnealingParams())
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
CosineAnnealingParams(last_epoch=-1, warmup_steps=None, warmup_ratio=None, min_lr=0.0)
|
||||
|
||||
Register scheduler
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
To register a new scheduler to be used with NeMo, run:
|
||||
|
||||
.. autofunction:: nemo.core.optim.lr_scheduler.register_scheduler
|
||||
|
||||
Save and Restore
|
||||
----------------
|
||||
|
||||
NeMo models all come with ``.save_to`` and ``.restore_from`` methods.
|
||||
|
||||
Save
|
||||
~~~~
|
||||
|
||||
To save a NeMo model, run:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
model.save_to('/path/to/model.nemo')
|
||||
|
||||
Everything needed to use the trained model is packaged and saved in the ``.nemo`` file. For example, in the NLP domain, ``.nemo`` files
|
||||
include the necessary tokenizer models and/or vocabulary files, etc.
|
||||
|
||||
.. note:: A ``.nemo`` file is simply an archive like any other ``.tar`` file.
|
||||
|
||||
Restore
|
||||
~~~~~~~
|
||||
|
||||
To restore a NeMo model, run:
|
||||
|
||||
.. code-block:: Python
|
||||
|
||||
model.restore_from('/path/to/model.nemo')
|
||||
|
||||
When using the PyTorch Lightning Trainer, a PyTorch Lightning checkpoint is created. These are mainly used within NeMo to auto-resume
|
||||
training. Since NeMo models are ``LightningModules``, the PyTorch Lightning method ``load_from_checkpoint`` is available. Note that
|
||||
``load_from_checkpoint`` won't necessarily work out-of-the-box for all models as some models require more artifacts than just the
|
||||
checkpoint to be restored. For these models, the user will have to override ``load_from_checkpoint`` if they want to use it.
|
||||
|
||||
It's highly recommended to use ``restore_from`` to load NeMo models.
|
||||
|
||||
Register Artifacts
|
||||
------------------
|
||||
|
||||
Conversational AI models can be complicated to restore as more information is needed than just the checkpoint weights in order to use the model.
|
||||
NeMo models can save additional artifacts in the .nemo file by calling ``.register_artifact``.
|
||||
When restoring NeMo models using ``.restore_from`` or ``.from_pretrained``, any artifacts that were registered will be available automatically.
|
||||
|
||||
As an example, consider an NLP model that requires a trained tokenizer model.
|
||||
The tokenizer model file can be automatically added to the .nemo file with the following:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
self.encoder_tokenizer = get_nmt_tokenizer(
|
||||
...
|
||||
tokenizer_model=self.register_artifact(config_path='encoder_tokenizer.tokenizer_model',
|
||||
src='/path/to/tokenizer.model',
|
||||
verify_src_exists=True),
|
||||
)
|
||||
|
||||
By default, ``.register_artifact`` will always return a path. If the model is being restored from a .nemo file,
|
||||
then that path will be to the artifact in the .nemo file. Otherwise, ``.register_artifact`` will return the local path specified by the user.
|
||||
|
||||
``config_path`` is the artifact key. It usually corresponds to a model configuration but does not have to.
|
||||
The model config that is packaged with the .nemo file will be updated according to the ``config_path`` key.
|
||||
In the above example, the model config will have
|
||||
|
||||
.. code-block:: YAML
|
||||
|
||||
encoder_tokenizer:
|
||||
...
|
||||
tokenizer_model: nemo:4978b28103264263a03439aaa6560e5e_tokenizer.model
|
||||
|
||||
``src`` is the path to the artifact and the base-name of the path will be used when packaging the artifact in the .nemo file.
|
||||
Each artifact will have a hash prepended to the basename of ``src`` in the .nemo file. This is to prevent collisions with basenames
|
||||
base-names that are identical (say when there are two or more tokenizers, both called `tokenizer.model`).
|
||||
The resulting .nemo file will then have the following file:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
4978b28103264263a03439aaa6560e5e_tokenizer.model
|
||||
|
||||
If ``verify_src_exists`` is set to ``False``, then the artifact is optional. This means that ``.register_artifact`` will return ``None``
|
||||
if the ``src`` cannot be found.
|
||||
|
||||
|
||||
|
||||
Experiment Manager
|
||||
==================
|
||||
|
||||
NeMo's Experiment Manager leverages PyTorch Lightning for model checkpointing, TensorBoard Logging, and Weights and Biases logging. The
|
||||
Experiment Manager is included by default in all NeMo example scripts.
|
||||
|
||||
To use the experiment manager simply call :class:`~nemo.utils.exp_manager.exp_manager` and pass in the PyTorch Lightning ``Trainer``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
exp_manager(trainer, cfg.get("exp_manager", None))
|
||||
|
||||
And is configurable via YAML with Hydra.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
exp_manager:
|
||||
exp_dir: /path/to/my/experiments
|
||||
name: my_experiment_name
|
||||
create_tensorboard_logger: True
|
||||
create_checkpoint_callback: True
|
||||
|
||||
Optionally, launch TensorBoard to view the training results in ``./nemo_experiments`` (by default).
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
tensorboard --bind_all --logdir nemo_experiments
|
||||
|
||||
..
|
||||
|
||||
If ``create_checkpoint_callback`` is set to ``True``, then NeMo automatically creates checkpoints during training
|
||||
using PyTorch Lightning's `ModelCheckpoint <https://pytorch-lightning.readthedocs.io/en/stable/extensions/generated/pytorch_lightning.callbacks.ModelCheckpoint.html#pytorch_lightning.callbacks.ModelCheckpoint>`_.
|
||||
We can configure the ``ModelCheckpoint`` via YAML or CLI.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
exp_manager:
|
||||
...
|
||||
# configure the PyTorch Lightning ModelCheckpoint using checkpoint_call_back_params
|
||||
# any ModelCheckpoint argument can be set here
|
||||
|
||||
# save the best checkpoints based on this metric
|
||||
checkpoint_callback_params.monitor=val_loss
|
||||
|
||||
# choose how many total checkpoints to save
|
||||
checkpoint_callback_params.save_top_k=5
|
||||
|
||||
We can auto-resume training as well by configuring the ``exp_manager``. Being able to auto-resume is important when doing long training
|
||||
runs that are premptible or may be shut down before the training procedure has completed. To auto-resume training, set the following
|
||||
via YAML or CLI:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
exp_manager:
|
||||
...
|
||||
# resume training if checkpoints already exist
|
||||
resume_if_exists: True
|
||||
|
||||
# to start training with no existing checkpoints
|
||||
resume_ignore_no_checkpoint: True
|
||||
|
||||
# by default experiments will be versioned by datetime
|
||||
# we can set our own version with
|
||||
exp_manager.version: my_experiment_version
|
||||
|
||||
Neural Modules
|
||||
==============
|
||||
|
||||
NeMo is built around Neural Modules, conceptual blocks of neural networks that take typed inputs and produce typed outputs. Such
|
||||
modules typically represent data layers, encoders, decoders, language models, loss functions, or methods of combining activations.
|
||||
NeMo makes it easy to combine and re-use these building blocks while providing a level of semantic correctness checking via its neural
|
||||
type system.
|
||||
|
||||
.. note:: *All Neural Modules inherit from ``torch.nn.Module`` and are therefore compatible with the PyTorch ecosystem.*
|
||||
|
||||
There are 3 types on Neural Modules:
|
||||
|
||||
- Regular modules
|
||||
- Dataset/IterableDataset
|
||||
- Losses
|
||||
|
||||
Every Neural Module in NeMo must inherit from `nemo.core.classes.module.NeuralModule` class.
|
||||
|
||||
.. autoclass:: nemo.core.classes.module.NeuralModule
|
||||
|
||||
Every Neural Modules inherits the ``nemo.core.classes.common.Typing`` interface and needs to define neural types for its inputs and outputs.
|
||||
This is done by defining two properties: ``input_types`` and ``output_types``. Each property should return an ordered dictionary of
|
||||
"port name"->"port neural type" pairs. Here is the example from :class:`~nemo.collections.asr.modules.ConvASREncoder` class:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def input_types(self):
|
||||
return OrderedDict(
|
||||
{
|
||||
"audio_signal": NeuralType(('B', 'D', 'T'), SpectrogramType()),
|
||||
"length": NeuralType(tuple('B'), LengthsType()),
|
||||
}
|
||||
)
|
||||
|
||||
@property
|
||||
def output_types(self):
|
||||
return OrderedDict(
|
||||
{
|
||||
"outputs": NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation()),
|
||||
"encoded_lengths": NeuralType(tuple('B'), LengthsType()),
|
||||
}
|
||||
)
|
||||
|
||||
@typecheck()
|
||||
def forward(self, audio_signal, length=None):
|
||||
...
|
||||
|
||||
The code snippet above means that ``nemo.collections.asr.modules.conv_asr.ConvASREncoder`` expects two arguments:
|
||||
* First one, named ``audio_signal`` of shape ``[batch, dimension, time]`` with elements representing spectrogram values.
|
||||
* Second one, named ``length`` of shape ``[batch]`` with elements representing lengths of corresponding signals.
|
||||
|
||||
It also means that ``.forward(...)`` and ``__call__(...)`` methods each produce two outputs:
|
||||
* First one, of shape ``[batch, dimension, time]`` but with elements representing encoded representation (``AcousticEncodedRepresentation`` class).
|
||||
* Second one, of shape ``[batch]``, corresponding to their lengths.
|
||||
|
||||
.. tip:: It is a good practice to define types and add ``@typecheck()`` decorator to your ``.forward()`` method after your module is ready for use by others.
|
||||
|
||||
.. note:: The outputs of ``.forward(...)`` method will always be of type ``torch.Tensor`` or container of tensors and will work with any other Pytorch code. The type information is attached to every output tensor.
|
||||
If tensors without types is passed to your module, it will not fail, however the types will not be checked. Thus, it is recommended to define input/output types for all your modules, starting with
|
||||
data layers and add ``@typecheck()`` decorator to them.
|
||||
|
||||
.. note:: To temporarily disable typechecking, you can enclose your code in ```with typecheck.disable_checks():``` statement.
|
||||
|
||||
Neural Types
|
||||
============
|
||||
|
||||
Motivation
|
||||
----------
|
||||
|
||||
Neural Types describe the semantics, axis order, and dimensions of a tensor. The purpose of this type system is to catch semantic and
|
||||
dimensionality errors during model creation and facilitate module re-use.
|
||||
|
||||
.. image:: whyntypes.gif
|
||||
:width: 900
|
||||
:alt: Neural Types Motivation
|
||||
|
||||
``NeuralType`` class
|
||||
--------------------
|
||||
|
||||
Neural Types perform semantic checks for modules and models inputs/outputs. They contain information about:
|
||||
|
||||
- Semantics of what is stored in the tensors. For example, logits, logprobs, audiosignal, embeddings, etc.
|
||||
- Axes layout, semantic and (optionally) dimensionality. For example: ``[Batch, Time, Channel]``
|
||||
|
||||
Types are implemented in ``nemo.core.neural_types.NeuralType`` class. When you instantiate an instance of this class, you
|
||||
are expected to include both *axes* information and *element type* information.
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.NeuralType
|
||||
|
||||
Type Comparison Results
|
||||
-----------------------
|
||||
|
||||
When comparing two neural types, the following comparison results are generated.
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.NeuralTypeComparisonResult
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Long vs short notation
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
NeMo's ``NeuralType`` class allows you to express axis semantics information in long and short form. Consider these two equivalent types. Both encoder 3 dimensional tensors and both contain elements of type ``AcousticEncodedRepresentation`` (this type is a typical output of ASR encoders).
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
long_version = NeuralType(
|
||||
axes=(AxisType(AxisKind.Batch, None), AxisType(AxisKind.Dimension, None), AxisType(AxisKind.Time, None)),
|
||||
elements_type=AcousticEncodedRepresentation(),
|
||||
)
|
||||
short_version = NeuralType(('B', 'D', 'T'), AcousticEncodedRepresentation())
|
||||
assert long_version.compare(short_version) == NeuralTypeComparisonResult.SAME
|
||||
|
||||
Transpose same
|
||||
~~~~~~~~~~~~~~
|
||||
|
||||
Often it is useful to know if a simple transposition will solve type incompatibility. This is the case if the comparison result of two types equals ``nemo.core.neural_types.NeuralTypeComparisonResult.TRANSPOSE_SAME``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
type1 = NeuralType(axes=('B', 'T', 'C'))
|
||||
type2 = NeuralType(axes=('T', 'B', 'C'))
|
||||
assert type1.compare(type2) == NeuralTypeComparisonResult.TRANSPOSE_SAME
|
||||
assert type2.compare(type1) == NeuralTypeComparisonResult.TRANSPOSE_SAME
|
||||
|
||||
Note that in this example, we dropped ``elements_type`` argument of ``NeuralType`` constructor. If not supplied, the element type is ``VoidType``.
|
||||
|
||||
``VoidType`` for elements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Sometimes it is useful to express that elements' types don't matter but axes layout do. ``VoidType`` for elements can be used to express this.
|
||||
|
||||
.. note:: ``VoidType`` is compatible with every other elements' type but not the other way around. See the following code snippet below for details.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
btc_spctr = NeuralType(('B', 'T', 'C'), SpectrogramType())
|
||||
btc_spct_bad = NeuralType(('B', 'T'), SpectrogramType())
|
||||
# Note the VoidType for elements here
|
||||
btc_void = NeuralType(('B', 'T', 'C'), VoidType())
|
||||
|
||||
# This is true because VoidType is compatible with every other element type (SpectrogramType in this case)
|
||||
# And axes layout between btc_void and btc_spctr is the same
|
||||
assert btc_void.compare(btc_spctr) == NeuralTypeComparisonResult.SAME
|
||||
# These two types are incompatible because even though VoidType is used for elements on one side,
|
||||
# the axes layout is different
|
||||
assert btc_void.compare(btc_spct_bad) == NeuralTypeComparisonResult.INCOMPATIBLE
|
||||
# Note that even though VoidType is compatible with every other type, other types are not compatible with VoidType!
|
||||
# It is one-way compatibility
|
||||
assert btc_spctr.compare(btc_void) == NeuralTypeComparisonResult.INCOMPATIBLE
|
||||
|
||||
Element type inheritance
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Neural types in NeMo support Python inheritance between element types. Consider an example where you want to develop a Neural Module which performs data augmentation for all kinds of spectrograms.
|
||||
In ASR, two types of spectrograms are frequently used: mel and mfcc. To express this, we will create 3 classes to express
|
||||
element's types: ``SpectrogramType``, ``MelSpectrogramType(SpectrogramType)``, ``MFCCSpectrogramType(SpectrogramType)``.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
input = NeuralType(('B', 'D', 'T'), SpectrogramType())
|
||||
out1 = NeuralType(('B', 'D', 'T'), MelSpectrogramType())
|
||||
out2 = NeuralType(('B', 'D', 'T'), MFCCSpectrogramType())
|
||||
|
||||
# MelSpectrogram and MFCCSpectrogram are not interchangeable.
|
||||
assert out1.compare(out2) == NeuralTypeComparisonResult.INCOMPATIBLE
|
||||
assert out2.compare(out1) == NeuralTypeComparisonResult.INCOMPATIBLE
|
||||
# Type comparison detects that MFCC/MelSpectrogramType is a kind of SpectrogramType and can be accepted.
|
||||
assert input.compare(out1) == NeuralTypeComparisonResult.GREATER
|
||||
assert input.compare(out2) == NeuralTypeComparisonResult.GREATER
|
||||
|
||||
Custom element types
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
It is possible to create user-defined element types to express the semantics of elements in your tensors. To do so, the user will need to inherit and implement abstract methods of the ``nemo.core.neural_types.elements.ElementType`` class
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.elements.ElementType
|
||||
|
||||
Note that element types can be parametrized. Consider this example where it distinguishes between audio sampled at 8Khz and 16Khz.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
audio16K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(16000))
|
||||
audio8K = NeuralType(axes=('B', 'T'), elements_type=AudioSignal(8000))
|
||||
|
||||
assert audio8K.compare(audio16K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS
|
||||
assert audio16K.compare(audio8K) == NeuralTypeComparisonResult.SAME_TYPE_INCOMPATIBLE_PARAMS
|
||||
|
||||
Enforcing dimensions
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
In addition to specifying tensor layout and elements' semantics, neural types also allow you to enforce tensor dimensions.
|
||||
The user will have to use long notations to specify dimensions. Short notations only allows you to specify axes semantics and assumes
|
||||
arbitrary dimensions.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
type1 = NeuralType(
|
||||
(AxisType(AxisKind.Batch, 64), AxisType(AxisKind.Time, 10), AxisType(AxisKind.Dimension, 128)),
|
||||
SpectrogramType(),
|
||||
)
|
||||
type2 = NeuralType(('B', 'T', 'C'), SpectrogramType())
|
||||
|
||||
# type2 will accept elements of type1 because their axes semantics match and type2 does not care about dimensions
|
||||
assert type2.compare(type1), NeuralTypeComparisonResult.SAME
|
||||
# type1 will not accept elements of type2 because it need dimensions to match strictly.
|
||||
assert type1.compare(type2), NeuralTypeComparisonResult.DIM_INCOMPATIBLE
|
||||
|
||||
Generic Axis kind
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Sometimes (especially in the case of loss modules) it is useful to be able to specify a "generic" axis kind which will make it
|
||||
compatible with any other kind of axis. This is easy to express with Neural Types by using ``nemo.core.neural_types.axes.AxisKind.Any`` for axes.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
type1 = NeuralType(('B', 'Any', 'Any'), SpectrogramType())
|
||||
type2 = NeuralType(('B', 'T', 'C'), SpectrogramType())
|
||||
type3 = NeuralType(('B', 'C', 'T'), SpectrogramType())
|
||||
|
||||
# type1 will accept elements of type2 and type3 because it only cares about element kind (SpectrogramType)
|
||||
# number of axes (3) and that first one corresponds to batch
|
||||
assert type1.compare(type2) == NeuralTypeComparisonResult.SAME
|
||||
assert type1.compare(type3) == NeuralTypeComparisonResult.INCOMPATIBLE
|
||||
|
||||
Container types
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
The NeMo-type system understands Python containers (lists). If your module returns a nested list of typed tensors, the way to express it is by
|
||||
using Python list notation and Neural Types together when defining your input/output types.
|
||||
|
||||
The example below shows how to express that your module returns single output ("out") which is list of lists of two dimensional tensors of shape ``[batch, dimension]`` containing logits.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
@property
|
||||
def output_types(self):
|
||||
return {
|
||||
"out": [[NeuralType(('B', 'D'), LogitsType())]],
|
||||
}
|
||||
|
||||
Core APIs
|
||||
=========
|
||||
|
||||
Base class for all NeMo models
|
||||
------------------------------
|
||||
|
||||
.. autoclass:: nemo.core.ModelPT
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
:undoc-members: cfg, num_weights
|
||||
:exclude-members: set_eff_save, use_eff_save, teardown
|
||||
|
||||
Base Neural Module class
|
||||
------------------------
|
||||
|
||||
.. autoclass:: nemo.core.NeuralModule
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
Neural Type classes
|
||||
-------------------
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.NeuralType
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.axes.AxisType
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.elements.ElementType
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
.. autoclass:: nemo.core.neural_types.comparison.NeuralTypeComparisonResult
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
Experiment manager
|
||||
------------------
|
||||
|
||||
.. autoclass:: nemo.utils.exp_manager.exp_manager
|
||||
:show-inheritance:
|
||||
:members:
|
||||
:member-order: bysource
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 302 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
@@ -0,0 +1,67 @@
|
||||
NVIDIA NeMo User Guide
|
||||
======================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Getting Started
|
||||
:name: starthere
|
||||
|
||||
starthere/intro
|
||||
starthere/tutorials
|
||||
starthere/best-practices
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: NeMo Core
|
||||
:name: core
|
||||
|
||||
core/core
|
||||
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Speech Processing
|
||||
:name: Speech Processing
|
||||
|
||||
asr/intro
|
||||
asr/speech_classification/intro
|
||||
asr/speaker_recognition/intro
|
||||
asr/speaker_diarization/intro
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Natural Language Processing
|
||||
:name: Natural Language Processing
|
||||
|
||||
nlp/intro
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Text To Speech
|
||||
:name: Text To Speech
|
||||
:includehidden:
|
||||
:titlesonly:
|
||||
|
||||
tts/intro
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Common
|
||||
:name: Common
|
||||
|
||||
common/intro
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Text Processing
|
||||
:name: Text Processing
|
||||
|
||||
nemo_text_processing/intro
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Tools
|
||||
:name: Tools
|
||||
|
||||
tools/intro
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
NeMo Text Processing API
|
||||
========================
|
||||
|
||||
Model Classes
|
||||
-------------
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.GraphFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.ClassifyFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.VerbalizeFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.VerbalizeFinalFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.inverse_text_normalization.ClassifyFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.inverse_text_normalization.VerbalizeFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.inverse_text_normalization.VerbalizeFinalFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
Text Processing
|
||||
===============
|
||||
|
||||
`nemo_text_processing` is a python package that is installed with the `nemo_toolkit`.
|
||||
|
||||
See :doc:`NeMo Introduction <../starthere/intro>` for installation details.
|
||||
|
||||
Additional requirements can be found in `setup.sh <https://github.com/NVIDIA/NeMo/blob/main/nemo_text_processing/setup.sh>`_.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
text_normalization
|
||||
inverse_text_normalization
|
||||
api
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
Inverse Text Normalization
|
||||
==========================
|
||||
|
||||
Inverse text normalization (ITN) is a part of the Automatic Speech Recognition (ASR) post-processing pipeline.
|
||||
ITN is the task of converting the raw spoken output of the ASR model into its written form to improve text readability.
|
||||
|
||||
For example,
|
||||
`"in nineteen seventy"` -> `"in 1975"`
|
||||
and `"it costs one hundred and twenty three dollars"` -> `"it costs $123"`.
|
||||
|
||||
NeMo ITN :cite:`textprocessing-itn-zhang2021nemo` is based on WFST-grammars :cite:`textprocessing-itn-Mohri2009`. We also provide a deployment route to C++ using `Sparrowhawk <https://github.com/google/sparrowhawk>`_ :cite:`textprocessing-itn-sparrowhawk` -- an open-source version of Google Kestrel :cite:`textprocessing-itn-ebden2015kestrel`.
|
||||
See :doc:`Text Procesing Deployment <../tools/text_processing_deployment>` for details.
|
||||
|
||||
.. note::
|
||||
|
||||
For more details, see the tutorial `NeMo/tutorials/text_processing/Inverse_Text_Normalization.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/text_processing/Inverse_Text_Normalization.ipynb>`__ in `Google's Colab <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/text_processing/Inverse_Text_Normalization.ipynb>`_.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Classes
|
||||
----------------------------------
|
||||
|
||||
|
||||
The base class for every grammar is :class:`GraphFst<nemo_text_processing.text_normalization.GraphFst>`.
|
||||
This tool is designed as a two-stage application: 1. `classification` of the input into semiotic tokens and 2. `verbalization` into written form.
|
||||
For every stage and every semiotic token class there is a corresponding grammar, e.g. :class:`taggers.CardinalFst<nemo_text_processing.inverse_text_normalization.taggers.cardinal.CardinalFst>`
|
||||
and :class:`verbalizers.CardinalFst<nemo_text_processing.inverse_text_normalization.verbalizers.cardinal.CardinalFst>`.
|
||||
Together, they compose the final grammars :class:`ClassifyFst<nemo_text_processing.inverse_text_normalization.ClassifyFst>` and
|
||||
:class:`VerbalizeFinalFst<nemo_text_processing.inverse_text_normalization.VerbalizeFinalFst>` that are compiled into WFST and used for inference.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
.. autoclass:: nemo_text_processing.inverse_text_normalization.ClassifyFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.inverse_text_normalization.VerbalizeFinalFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
|
||||
Prediction
|
||||
----------------------------------
|
||||
|
||||
Example prediction run:
|
||||
|
||||
.. code::
|
||||
|
||||
python run_prediction.py --input=<INPUT_TEXT_FILE> --output=<OUTPUT_PATH> [--verbose]
|
||||
|
||||
|
||||
The input is expected to be lower-cased. Punctuation are outputted with separating spaces after semiotic tokens, e.g. `"i see, it is ten o'clock..."` -> `"I see, it is 10:00 . . ."`.
|
||||
Inner-sentence white-space characters in the input are not maintained.
|
||||
|
||||
|
||||
|
||||
Data Cleaning for Evaluation
|
||||
----------------------------------
|
||||
|
||||
.. code::
|
||||
|
||||
python clean_eval_data.py --input=<INPUT_TEXT_FILE>
|
||||
|
||||
|
||||
|
||||
Evaluation
|
||||
----------------------------------
|
||||
|
||||
Example evaluation run on (cleaned) `Google's text normalization dataset <https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish>`__ :cite:`textprocessing-itn-sproat2016rnn`:
|
||||
|
||||
.. code::
|
||||
|
||||
python run_evaluation.py --input=./en_with_types/output-00001-of-00100 [--cat CLASS_CATEGORY] [--filter]
|
||||
|
||||
|
||||
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: textprocessing_all.bib
|
||||
:style: plain
|
||||
:labelprefix: TEXTPROCESSING-ITN
|
||||
:keyprefix: textprocessing-itn-
|
||||
@@ -0,0 +1,78 @@
|
||||
Text Normalization
|
||||
==================
|
||||
|
||||
NeMo Text Normalization converts text from written form into its verbalized form. It is used as a preprocessing step before Text to Speech (TTS). It could also be used for preprocessing Automatic Speech Recognition (ASR) training transcripts.
|
||||
|
||||
|
||||
For example,
|
||||
`"at 10:00"` -> `"at ten o'clock"`
|
||||
and `"it weighs 10kg."` -> `"it weights ten kilograms ."`.
|
||||
|
||||
|
||||
NeMo Text Normalization :cite:`textprocessing-norm-zhang2021nemo` is based on WFST-grammars :cite:`textprocessing-norm-Mohri2009`. We also provide a deployment route to C++ using `Sparrowhawk <https://github.com/google/sparrowhawk>`_ :cite:`textprocessing-norm-sparrowhawk` -- an open-source version of Google Kestrel :cite:`textprocessing-norm-ebden2015kestrel`.
|
||||
See :doc:`Text Procesing Deployment <../tools/text_processing_deployment>` for details.
|
||||
|
||||
|
||||
.. note::
|
||||
|
||||
For more details, see the tutorial `NeMo/tutorials/text_processing/Text_Normalization.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/text_processing/Text_Normalization.ipynb>`__ in `Google's Colab <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/text_processing/Text_Normalization.ipynb>`_.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Classes
|
||||
----------------------------------
|
||||
|
||||
|
||||
The base class for every grammar is :class:`GraphFst<nemo_text_processing.text_normalization.GraphFst>`.
|
||||
This tool is designed as a two-stage application: 1. `classification` of the input into semiotic tokens and 2. `verbalization` into written form.
|
||||
For every stage and every semiotic token class there is a corresponding grammar, e.g. :class:`taggers.CardinalFst<nemo_text_processing.text_normalization.taggers.cardinal.CardinalFst>`
|
||||
and :class:`verbalizers.CardinalFst<nemo_text_processing.text_normalization.verbalizers.cardinal.CardinalFst>`.
|
||||
Together, they compose the final grammars :class:`ClassifyFst<nemo_text_processing.text_normalization.ClassifyFst>` and
|
||||
:class:`VerbalizeFinalFst<nemo_text_processing.text_normalization.VerbalizeFinalFst>` that are compiled into WFST and used for inference.
|
||||
|
||||
|
||||
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.ClassifyFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo_text_processing.text_normalization.VerbalizeFinalFst
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
|
||||
Prediction
|
||||
----------------------------------
|
||||
|
||||
Example prediction run:
|
||||
|
||||
.. code::
|
||||
|
||||
python run_prediction.py <--input INPUT_TEXT_FILE> <--output OUTPUT_PATH> [--input_case INPUT_CASE]
|
||||
|
||||
``INPUT_CASE`` specifies whether to treat the input as lower-cased or case sensitive. By default treat the input as cased since this is more informative, especially for abbreviations. Punctuation are outputted with separating spaces after semiotic tokens, e.g. `"I see, it is 10:00..."` -> `"I see, it is ten o'clock . . ."`.
|
||||
Inner-sentence white-space characters in the input are not maintained.
|
||||
|
||||
|
||||
Evaluation
|
||||
----------------------------------
|
||||
|
||||
Example evaluation run on `Google's text normalization dataset <https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish>`__ :cite:`textprocessing-norm-sproat2016rnn`:
|
||||
|
||||
.. code::
|
||||
|
||||
python run_evaluation.py --input=./en_with_types/output-00001-of-00100 [--cat CLASS_CATEGORY] [--input_case INPUT_CASE]
|
||||
|
||||
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: textprocessing_all.bib
|
||||
:style: plain
|
||||
:labelprefix: TEXTPROCESSING-NORM
|
||||
:keyprefix: textprocessing-norm-
|
||||
@@ -0,0 +1,58 @@
|
||||
@article{ebden2015kestrel,
|
||||
title={The Kestrel TTS text normalization system},
|
||||
author={Ebden, Peter and Sproat, Richard},
|
||||
journal={Natural Language Engineering},
|
||||
volume={21},
|
||||
number={3},
|
||||
pages={333},
|
||||
year={2015},
|
||||
publisher={Cambridge University Press}
|
||||
}
|
||||
|
||||
|
||||
@article{sproat2016rnn,
|
||||
title={RNN approaches to text normalization: A challenge},
|
||||
author={Sproat, Richard and Jaitly, Navdeep},
|
||||
journal={arXiv preprint arXiv:1611.00068},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@book{taylor2009text,
|
||||
title={Text-to-speech synthesis},
|
||||
author={Taylor, Paul},
|
||||
year={2009},
|
||||
publisher={Cambridge university press}
|
||||
}
|
||||
|
||||
@misc{zhang2021nemo,
|
||||
title={NeMo Inverse Text Normalization: From Development To Production},
|
||||
author={Yang Zhang and Evelina Bakhturina and Kyle Gorman and Boris Ginsburg},
|
||||
year={2021},
|
||||
eprint={2104.05055},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
|
||||
@Inbook{Mohri2009,
|
||||
author="Mohri, Mehryar",
|
||||
editor="Droste, Manfred
|
||||
and Kuich, Werner
|
||||
and Vogler, Heiko",
|
||||
title="Weighted Automata Algorithms",
|
||||
bookTitle="Handbook of Weighted Automata",
|
||||
year="2009",
|
||||
publisher="Springer Berlin Heidelberg",
|
||||
address="Berlin, Heidelberg",
|
||||
pages="213--254",
|
||||
abstract="Weighted automata and transducers are widely used in modern applications in bioinformatics and text, speech, and image processing. This chapter describes several fundamental weighted automata and shortest-distance algorithms including composition, determinization, minimization, and synchronization, as well as single-source and all-pairs shortest distance algorithms over general semirings. It presents the pseudocode of these algorithms, gives an analysis of their running time complexity, and illustrates their use in some simple cases. Many other complex weighted automata and transducer algorithms used in practice can be obtained by combining these core algorithms.",
|
||||
isbn="978-3-642-01492-5",
|
||||
doi="10.1007/978-3-642-01492-5_6",
|
||||
url="https://doi.org/10.1007/978-3-642-01492-5_6"
|
||||
}
|
||||
|
||||
@inproceedings{sparrowhawk,
|
||||
title = {TTS for Low Resource Languages: A Bangla Synthesizer},
|
||||
author = {Alexander Gutkin and Linne Ha and Martin Jansche and Knot Pipatsrisawat and Richard Sproat},
|
||||
booktitle = {10th Language Resources and Evaluation Conference},
|
||||
year = {2016},
|
||||
}
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
NeMo NLP collection API
|
||||
=======================
|
||||
|
||||
Model Classes
|
||||
-------------
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.TextClassificationModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact, classifytext
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.GLUEModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.PunctuationCapitalizationModel
|
||||
:show-inheritance:
|
||||
:members: add_punctuation_capitalization, setup_training_data, setup_optimization, setup_validation_data, setup_test_data, multi_validation_epoch_end, register_artifact
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.TokenClassificationModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, register_artifact
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.QAModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization, setup_validation_data, setup_test_data, inference, validation_epoch_end, test_epoch_end
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.models.BERTLMModel
|
||||
:show-inheritance:
|
||||
:members: setup_training_data, setup_optimization
|
||||
|
||||
Modules
|
||||
-------
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.BertModule
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.MegatronBertEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.AlbertEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.BertEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.DistilBertEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.RobertaEncoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.SequenceClassifier
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.SequenceRegression
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.nlp.modules.SequenceTokenClassifier
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autofunction:: nemo.collections.nlp.modules.get_lm_model
|
||||
|
||||
.. autofunction:: nemo.collections.nlp.modules.get_pretrained_lm_models_list
|
||||
|
||||
.. autofunction:: nemo.collections.nlp.modules.get_megatron_lm_models_list
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
.. _bert_pretraining:
|
||||
|
||||
BERT
|
||||
====
|
||||
|
||||
BERT is an autoencoding language model with a final loss composed of:
|
||||
|
||||
- masked language model loss
|
||||
- next sentence prediction
|
||||
|
||||
The model architecture is published in `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ :cite:`nlp-bert-devlin2018bert`.
|
||||
The model is originally trained on English Wikipedia and BookCorpus. BERT is often used as a language model encoder for downstream tasks, for example, :ref:`token_classification`, :ref:`text_classification`, :ref:`question_answering`, etc.
|
||||
Domain-specific BERT models can be advantageous for a wide range of applications. One notable application is the domain-specific BERT in a biomedical setting,
|
||||
e.g. BioBERT :cite:`nlp-bert-lee2019biobert` or its improved derivative BioMegatron :cite:`nlp-bert-shin2020biomegatron`. For the latter, refer to :ref:`megatron_finetuning`.
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.collections.nlp.models import BERTLMModel
|
||||
|
||||
# to get the list of pre-trained models
|
||||
BERTLMModel.list_available_models()
|
||||
|
||||
# Download and load the pre-trained BERT-based model
|
||||
model = BERTLMModel.from_pretrained("bertbaseuncased")
|
||||
|
||||
Available Models
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. list-table:: *Pretrained Models*
|
||||
:widths: 5 10
|
||||
:header-rows: 1
|
||||
|
||||
* - Model
|
||||
- Pretrained Checkpoint
|
||||
* - BERT-base uncased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:bertbaseuncased
|
||||
* - BERT-large uncased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:bertlargeuncased
|
||||
|
||||
.. _dataset_bert_pretraining:
|
||||
|
||||
Data Input for the BERT model
|
||||
-----------------------------
|
||||
|
||||
Data preprocessing can be either done on-the-fly during training or offline before training. The latter is optimized and recommended
|
||||
for large text corpora. This was also used in the original paper to train the model on Wikipedia and BookCorpus. For on-the-fly data
|
||||
processing, provide text files with sentences for training and validation, where words are separated by spaces, i.e.: ``[WORD] [SPACE] [WORD] [SPACE] [WORD]``.
|
||||
To use this pipeline in training, use the dedicated configuration file `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml`.
|
||||
|
||||
To process data offline in advance, refer to the `BERT Quick Start Guide <https://github.com/NVIDIA/DeepLearningExamples/tree/master/PyTorch/LanguageModeling/BERT#quick-start-guide>`__.
|
||||
To recreate the original Wikipedia and BookCorpus datasets, follow steps 1-5 in the Quick Start Guide and run the script ``./data/create_datasets_from_start.sh`` inside the Docker container.
|
||||
The ``downloaded`` folder should include two sub folders ``lower_case_[0,1]_seq_len_128_max_pred_20_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5``
|
||||
and ``lower_case_[0,1]_seq_len_512_max_pred_80_masked_lm_prob_0.15_random_seed_12345_dupe_factor_5``, containing sequences of length 128 with a maximum of 20 masked tokens
|
||||
and sequences of length 512 with a maximum of 80 masked tokens respectively. To use this pipeline in training, use the dedicated configuration file ``NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml``
|
||||
and specify the path to the created hd5f files.
|
||||
|
||||
|
||||
Training the BERT model
|
||||
-----------------------
|
||||
|
||||
Example of model configuration for on-the-fly data preprocessing: `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml>`__.
|
||||
Example of model configuration for offline data preprocessing: `NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml>`__.
|
||||
|
||||
The specification can be grouped into three categories:
|
||||
|
||||
- Parameters that describe the training process: **trainer**
|
||||
- Parameters that describe the datasets: **model.train_ds**, **model.validation_ds**
|
||||
- Parameters that describe the model: **model**, **model.tokenizer**, **model.language_model**
|
||||
|
||||
More details about parameters in the config file can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.only_mlm_loss** | bool | Only uses masked language model without next sentence prediction. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **train_ds.data_file** | string | Name of the text file or hdf5 data directory. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **train_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
More details about parameters for offline data preprocessing can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **train_ds.max_predictions_per_seq** | integer | Maximum number of masked tokens in a sequence in the preprocessed data. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
More details about parameters for online data preprocessing can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.max_seq_length** | integer | The maximum total input sequence length after tokenization. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.mask_prob** | float | Probability of masking a token in the input text during data processing. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.short_seq_prob** | float | Probability of having a sequence shorter than the maximum sequence length. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
.. note::
|
||||
|
||||
For offline data preprocessing, **model.tokenizer** is null. For downstream task, use the same tokenizer that was used for
|
||||
offline preprocessing. For online data preprocessing, **model.tokenizer** needs to be specified. See also :ref:`nlp_model` for
|
||||
details.
|
||||
|
||||
Example of the command for training the model:
|
||||
|
||||
.. code::
|
||||
|
||||
python bert_pretraining.py \
|
||||
model.train_ds.data_file=<PATH_TO_DATA> \
|
||||
trainer.max_epochs=<NUM_EPOCHS> \
|
||||
trainer.gpus=[<CHANGE_TO_GPU(s)_YOU_WANT_TO_USE>]
|
||||
|
||||
|
||||
Fine-tuning on Downstream Tasks
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
To use a trained BERT model checkpoint on a NeMo NLP downstream task, e.g. :ref:`question_answering`, specify
|
||||
:code:`model.language_model.lm_checkpoint=<PATH_TO_CHECKPOINT>`.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-BERT
|
||||
:keyprefix: nlp-bert-
|
||||
@@ -0,0 +1,10 @@
|
||||
.. _glue_benchmark:
|
||||
|
||||
GLUE Benchmark
|
||||
==============
|
||||
|
||||
We recommend you try the GLUE Benchmark model in a Jupyter notebook (can run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_): `NeMo/tutorials/nlp/GLUE_Benchmark.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/GLUE_Benchmark.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator).
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/glue_benchmark/glue_benchmark.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/glue_benchmark/glue_benchmark.py>`__.
|
||||
@@ -0,0 +1,10 @@
|
||||
.. _information_retrieval:
|
||||
|
||||
Information Retrieval
|
||||
=====================
|
||||
|
||||
We recommend you try the Information Retrieval model in a Jupyter notebook (can run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_): `NeMo/tutorials/nlp/Information_Retrieval_MSMARCO.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Information_Retrieval_MSMARCO.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for hardware the accelerator),
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/information_retrieval <https://github.com/NVIDIA/NeMo/tree/main/examples/nlp/information_retrieval>`__.
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
Natural Language Processing (NLP)
|
||||
=================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
models
|
||||
megatron_finetuning
|
||||
api
|
||||
|
||||
|
||||
Example scripts for the NLP collection can be found under `NeMo/example/nlp/ <https://github.com/NVIDIA/NeMo/tree/main/examples/nlp>`__.
|
||||
|
||||
NLP notebook-tutorials are located under `NeMo/tutorials/nlp/ <https://github.com/NVIDIA/NeMo/tree/main/tutorials/nlp>`__. Most NeMo
|
||||
tutorials can be run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.), with more details in the :ref:`tutorials` section.
|
||||
@@ -0,0 +1,244 @@
|
||||
.. _intent_slot:
|
||||
|
||||
Joint Intent and Slot Classification
|
||||
====================================
|
||||
|
||||
Joint Intent and Slot classification is a NLU task for classifying an intent and detecting all
|
||||
relevant slots (Entities) for the intent in a query. For example, in the query ``What is the weather in Santa Clara tomorrow morning?``,
|
||||
we would like to classify the query as a ``weather intent``, detect ``Santa Clara`` as a `location slot`,
|
||||
and ``tomorrow morning`` as a ``date_time slot``. Intent and Slot names are usually task-specific and
|
||||
defined as labels in the training data. This is a fundamental step that is executed in any
|
||||
task-driven conversational assistant.
|
||||
|
||||
Our BERT-based model implementation allows you to train and detect both of these tasks together.
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend you try the Joint Intent and Slot Classification model in a Jupyter notebook (can run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.): `NeMo/tutorials/nlp/Joint_Intent_and_Slot_Classification.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Joint_Intent_and_Slot_Classification.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator).
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/intent_slot_classification <https://github.com/NVIDIA/NeMo/tree/main/examples/nlp/intent_slot_classification>`__.
|
||||
|
||||
|
||||
NeMo Data Format
|
||||
----------------
|
||||
|
||||
When training the model, the dataset should be first converted to the required data format, which requires the following files:
|
||||
|
||||
- :code:`dict.intents.csv` - A list of all intent names in the data. One line per an intent name. The index of the intent line
|
||||
(starting from ``0``) is used to identify the appropriate intent in ``train.tsv`` and ``test.tsv`` files.
|
||||
|
||||
.. code::
|
||||
|
||||
weather
|
||||
alarm
|
||||
meeting
|
||||
...
|
||||
|
||||
- :code:`dict.slots.csv` - A list of all slot names in the data. One line per slot name. The index of the slot line
|
||||
(starting from ``0``) is used to identify the appropriate slots in the queries in ``train_slot.tsv`` and ``test_slot.tsv`` files.
|
||||
In the last line of this dictionary ``O`` slot name is used to identify all ``out of scope`` slots, which are usually the majority of the tokens
|
||||
in the queries.
|
||||
|
||||
.. code::
|
||||
|
||||
date
|
||||
time
|
||||
city
|
||||
...
|
||||
O
|
||||
|
||||
- :code:`train.tsv/test.tsv` - A list of original queries, one per line, with the intent number
|
||||
separated by a tab (e.g. "what alarms do i have set right now <TAB> 0"). Intent numbers are
|
||||
set according to the intent line in the intent dictionary file (:code:`dict.intents.csv`),
|
||||
starting from ``0``. The first line in these files should contain the header line ``sentence
|
||||
<tab> label``.
|
||||
|
||||
- :code:`train_slot.tvs/test_slot.tsv` - A list that contains one line per query, when each word from the original text queries
|
||||
is replaced by a token number from the slots dictionary file (``dict.slots.csv``), counted starting from ``0``. All the words
|
||||
which do not contain a relevant slot are replaced by ``out-of scope`` token number, which is also a part of the slot dictionary file,
|
||||
usually as the last entry there. For example a line from these files should look similar to: "54 0 0 54 54 12 12" (the numbers are
|
||||
separated by a space). These files do not contain a header line.
|
||||
|
||||
|
||||
Dataset Conversion
|
||||
------------------
|
||||
|
||||
To convert to the format of the model data, use the ``import_datasets`` utility, which implements
|
||||
the conversion for the Assistant dataset. Download the dataset `here <https://github.com/xliuhw/NLU-Evaluation-Data>`_ or you can
|
||||
write your own converter for the format that you are using for data annotation.
|
||||
|
||||
For a dataset that follows your own annotation format, we recommend using one text file for all
|
||||
samples of the same intent, with the name of the file as the name of the intent. Use one line per
|
||||
query, with brackets to define slot names. This is very similar to the assistant format, and you can
|
||||
adapt this converter utility or your own format with small changes:
|
||||
|
||||
::
|
||||
|
||||
did i set an alarm to [alarm_type : wake up] in the [timeofday : morning]
|
||||
|
||||
Run the ``dataset_converter`` command:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/intent_slot_classification/data/import_datasets.py
|
||||
--source_data_dir=`source_data_dir` \
|
||||
--target_data_dir=`target_data_dir` \
|
||||
--dataset_name=['assistant'|'snips'|'atis']
|
||||
|
||||
- :code:`source_data_dir`: the directory location of the your dataset
|
||||
- :code:`target_data_dir`: the directory location where the converted dataset should be saved
|
||||
- :code:`dataset_name`: one of the implemented dataset names
|
||||
|
||||
After conversion, ``target_data_dir`` should contain the following files:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--target_data_dir
|
||||
|-- dict.intents.csv
|
||||
|-- dict.slots.csv
|
||||
|-- train.tsv
|
||||
|-- train_slots.tsv
|
||||
|-- test.tsv
|
||||
|-- test_slots.tsv
|
||||
|
||||
Model Training
|
||||
--------------
|
||||
|
||||
This is a pretrained BERT based model with 2 linear classifier heads on the top of it, one for classifying an intent of the query and
|
||||
another for classifying slots for each token of the query. This model is trained with the combined loss function on the Intent and Slot
|
||||
classification task on the given dataset. The model architecture is based on the paper `BERT for Joint Intent Classification and Slot Filling <https://arxiv.org/pdf/1902.10909.pdf>`__:cite:`nlp-jis-chen2019bert`.
|
||||
|
||||
For each query, the model classifies it as one the intents from the intent dictionary and for each word of the query it will classify
|
||||
it as one of the slots from the slot dictionary, including out of scope slot for all the remaining words in the query which does not
|
||||
fall in another slot category. Out of scope slot (``O``) is a part of slot dictionary that the model is trained on.
|
||||
|
||||
Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/intent_slot_classification/conf/intent_slot_classification.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/intent_slot_classification/conf/intent_slot_classification_config.yaml>`__.
|
||||
In the configuration file, define the parameters of the training and the model, although most of the default values will work well.
|
||||
|
||||
The specification can be roughly grouped into three categories:
|
||||
|
||||
- Parameters that describe the training process: **trainer**
|
||||
- Parameters that describe the model: **model**
|
||||
- Parameters that describe the datasets: **model.train_ds**, **model.validation_ds**, **model.test_ds**,
|
||||
|
||||
More details about parameters in the spec file can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.data_dir** | string | -- | The path of the data converted to the specified format. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.class_balancing** | string | ``null`` | Choose from ``[null, weighted_loss]``. The ``weighted_los``s enables weighted class balancing of the loss. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.intent_loss_weight** | float | ``0.6`` | The elation of intent-to-slot loss in the total loss. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.pad_label** | integer | ``-1`` | A value to pad the inputs. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.ignore_extra_tokens** | boolean | ``false`` | A flag that specifies whether to ignore extra tokens. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.ignore_start_end** | boolean | ``true`` | A flag that specifies whether to not use the first and last token for slot training. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.head.num_output_layers** | integer | ``2`` | The number of fully connected layers of the classifier on top of the BERT model. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.head.fc_dropout** | float | ``0.1`` | The dropout ratio of the fully connected layers. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.prefix** | string | ``train`` | A prefix for the training file names. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.prefix** | string | ``dev`` | A prefix for the validation file names. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **test_ds.prefix** | string | ``test`` | A prefix for the test file names. |
|
||||
+-------------------------------------------+-----------------+----------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
For additional config parameters common to all NLP models, refer to the `nlp_model doc <https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/nlp_model.rst#model-nlp>`__.
|
||||
|
||||
The following is an example of the command for training the model:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/intent_slot_classification/intent_slot_classification.py
|
||||
model.data_dir=<PATH_TO_DATA_DIR> \
|
||||
trainer.max_epochs=<NUM_EPOCHS> \
|
||||
trainer.gpus=[<CHANGE_TO_GPU(s)_YOU_WANT_TO_USE>]
|
||||
|
||||
|
||||
Required Arguments for Training
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`model.data_dir`: the dataset directory
|
||||
|
||||
|
||||
Optional Arguments
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Most of the default parameters in the existing configuration file are already set appropriately, however, there are some parameters
|
||||
you may want to experiment with.
|
||||
|
||||
- ``trainer.max_epochs``: the number of training epochs (reasonable to be between 10 to 100)
|
||||
- ``model.class_balancing`` - value ``weighted_loss`` may help to train the model when there is unbalanced set of classes
|
||||
- ``model.intent_loss_weight`` - a number between 0 to 1 that defines a weight of the intent lost versus a slot loss during training. A
|
||||
default value 0.6 gives a slight preference for the intent lose optimization.
|
||||
|
||||
Training Procedure
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
At the start of evaluation, NeMo will print out a log of the experiment specification, a summary of the training dataset, and the
|
||||
model architecture.
|
||||
|
||||
As the model starts training, you should see a progress bar per epoch. During training, after each epoch, NeMo will display accuracy
|
||||
metrics on the validation dataset for every intent and slot separately, as well as the total accuracy. You can expect these numbers
|
||||
to grow up to 50-100 epochs, depending on the size of the trained data. Since this is a joint iIntent and slot training, usually
|
||||
intent's accuracy will grow first for the initial 10-20 epochs, and after that, slot's accuracy will start improving as well.
|
||||
|
||||
At the end of training, NeMo saves the best checkpoint on the validation dataset at the path specified by the experiment spec file
|
||||
before finishing.
|
||||
|
||||
.. code::
|
||||
|
||||
GPU available: True, used: True
|
||||
TPU available: None, using: 0 TPU cores
|
||||
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2]
|
||||
[NeMo W 2021-01-28 14:52:19 exp_manager:299] There was no checkpoint folder at checkpoint_dir :results/checkpoints. Training from scratch.
|
||||
[NeMo I 2021-01-28 14:52:19 exp_manager:186] Experiments will be logged at results
|
||||
...
|
||||
label precision recall f1 support
|
||||
weather.weather (label_id: 0) 0.00 0.00 0.00 128
|
||||
weather.temperature (label_id: 1) 0.00 0.00 0.00 0
|
||||
weather.temperature_yes_no (label_id: 2) 0.00 0.00 0.00 0
|
||||
weather.rainfall (label_id: 3) 0.00 0.00 0.00 0
|
||||
weather.rainfall_yes_no (label_id: 4) 0.00 0.00 0.00 0
|
||||
weather.snow (label_id: 5) 0.00 0.00 0.00 0
|
||||
weather.snow_yes_no (label_id: 6) 0.00 0.00 0.00 0
|
||||
weather.humidity (label_id: 7) 0.00 0.00 0.00 0
|
||||
weather.humidity_yes_no (label_id: 8) 0.00 0.00 0.00 0
|
||||
weather.windspeed (label_id: 9) 0.00 0.00 0.00 0
|
||||
weather.sunny (label_id: 10) 0.00 0.00 0.00 0
|
||||
weather.cloudy (label_id: 11) 0.00 0.00 0.00 0
|
||||
weather.alert (label_id: 12) 0.00 0.00 0.00 0
|
||||
context.weather (label_id: 13) 0.00 0.00 0.00 0
|
||||
context.continue (label_id: 14) 0.00 0.00 0.00 0
|
||||
context.navigation (label_id: 15) 0.00 0.00 0.00 0
|
||||
context.rating (label_id: 16) 0.00 0.00 0.00 0
|
||||
context.distance (label_id: 17) 0.00 0.00 0.00 0
|
||||
-------------------
|
||||
micro avg 0.00 0.00 0.00 128
|
||||
macro avg 0.00 0.00 0.00 128
|
||||
weighted avg 0.00 0.00 0.00 128
|
||||
|
||||
Model Evaluation and Inference
|
||||
------------------------------
|
||||
|
||||
There is no separate script for the evaluation and inference of this model in NeMo, however, inside of the example file `examples/nlp/intent_slot_classification/intent_slot_classification.py`
|
||||
after the training part is finished, you can see the code that evaluates the trained model on an evaluation test set and then an example of doing inference using a list of given queries.
|
||||
|
||||
For the deployment in the production environment, refer to `NVIDIA Jarvis <https://developer.nvidia.com/nvidia-jarvis-getting-started>`__ and `NVIDIA TLT documentation <https://docs.nvidia.com/metropolis/TLT/tlt-user-guide/text/nlp/index.html>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-JIS
|
||||
:keyprefix: nlp-jis-
|
||||
@@ -0,0 +1,6 @@
|
||||
.. _language_modeling:
|
||||
|
||||
Language Modeling
|
||||
=================
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/language_modeling <https://github.com/NVIDIA/NeMo/tree/main/examples/nlp/language_modeling>`__.
|
||||
@@ -0,0 +1,466 @@
|
||||
.. _machine_translation:
|
||||
|
||||
Machine Translation Models
|
||||
==========================
|
||||
Machine translation is the task of translating text from one language to another. For example, from English to Spanish. Models are
|
||||
based on the Transformer sequence-to-sequence architecture :cite:`nlp-machine_translation-vaswani2017attention`.
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/machine_translation/enc_dec_nmt.py <https://github.com/NVIDIA/NeMo/blob/v1.0.2/examples/nlp/machine_translation/enc_dec_nmt.py>`__.
|
||||
The default configuration file for the model can be found at: `NeMo/examples/nlp/machine_translation/conf/aayn_base.yaml <https://github.com/NVIDIA/NeMo/blob/v1.0.2/examples/nlp/machine_translation/conf/aayn_base.yaml>`__.
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.collections.nlp.models import MTEncDecModel
|
||||
|
||||
# To get the list of pre-trained models
|
||||
MTEncDecModel.list_available_models()
|
||||
|
||||
# Download and load the a pre-trained to translate from English to Spanish
|
||||
model = MTEncDecModel.from_pretrained("nmt_en_es_transformer12x2")
|
||||
|
||||
# Translate a sentence or list of sentences
|
||||
translations = model.translate(["Hello!"], source_lang="en", target_lang="es")
|
||||
|
||||
Available Models
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. list-table:: *Pretrained Models*
|
||||
:widths: 5 10
|
||||
:header-rows: 1
|
||||
|
||||
* - Model
|
||||
- Pretrained Checkpoint
|
||||
* - English -> German
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_de_transformer12x2
|
||||
* - German -> English
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_de_en_transformer12x2
|
||||
* - English -> Spanish
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_es_transformer12x2
|
||||
* - Spanish -> English
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_es_en_transformer12x2
|
||||
* - English -> French
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_fr_transformer12x2
|
||||
* - French -> English
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_fr_en_transformer12x2
|
||||
* - English -> Russian
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_ru_transformer6x6
|
||||
* - Russian -> English
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_ru_en_transformer6x6
|
||||
* - English -> Chinese
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_en_zh_transformer6x6
|
||||
* - Chinese -> English
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:nmt_zh_en_transformer6x6
|
||||
|
||||
Data Format
|
||||
-----------
|
||||
|
||||
Supervised machine translation models require parallel corpora which comprises many examples of sentences in a source language and
|
||||
their corresponding translation in a target language. We use parallel data formatted as seperate text files for source and target
|
||||
languages where sentences in corresponding files are aligned like in the table below.
|
||||
|
||||
.. list-table:: *Parallel Coprus*
|
||||
:widths: 10 10
|
||||
:header-rows: 1
|
||||
|
||||
* - train.english.txt
|
||||
- train.spanish.txt
|
||||
* - Hello .
|
||||
- Hola .
|
||||
* - Thank you .
|
||||
- Gracias .
|
||||
* - You can now translate from English to Spanish in NeMo .
|
||||
- Ahora puedes traducir del inglés al español en NeMo .
|
||||
|
||||
It is common practice to apply data cleaning, normalization, and tokenization to the data prior to training a translation model and
|
||||
NeMo expects already cleaned, normalized, and tokenized data. The only data pre-processing NeMo does is subword tokenization with BPE
|
||||
:cite:`nlp-machine_translation-sennrich2015neural`.
|
||||
|
||||
Data Cleaning, Normalization & Tokenization
|
||||
-------------------------------------------
|
||||
|
||||
We recommend applying the following steps to clean, normalize, and tokenize your data. All pre-trained models released, apply these data pre-processing steps.
|
||||
|
||||
#. Language ID filtering - This step filters out examples from your training dataset that aren't in the correct language. For example,
|
||||
many datasets contain examples where source and target sentences are in the same language. You can use a pre-trained language ID
|
||||
classifier from `fastText <https://fasttext.cc/docs/en/language-identification.html>`__. You can then run our script using the
|
||||
``lid.176.bin`` model downloaded from the fastText website.
|
||||
|
||||
.. code ::
|
||||
|
||||
python NeMo/scripts/neural_machine_translation/filter_by_language.py \
|
||||
--input-src train.en \
|
||||
--input-tgt train.es \
|
||||
--output-src train_lang_filtered.en \
|
||||
--output-tgt train_lang_filtered.es \
|
||||
--source-lang en \
|
||||
--target-lang es \
|
||||
--removed-src train_noise.en \
|
||||
--removed-tgt train_noise.de \
|
||||
--fasttext-model lid.176.bin
|
||||
|
||||
#. Length filtering - We filter out sentences from the data that are below a minimum length (1) or exceed a maximum length (250). We
|
||||
also filter out sentences where the ratio between source and target lengths exceeds 1.3 except for English <-> Chinese models.
|
||||
`Moses <https://github.com/moses-smt/mosesdecoder>`__ is a statistical machine translation toolkit that contains many useful
|
||||
pre-processing scripts.
|
||||
|
||||
.. code ::
|
||||
|
||||
perl mosesdecoder/scripts/training/clean-corpus-n.perl -ratio 1.3 train en es train.filter 1 250
|
||||
|
||||
#. Data cleaning - While language ID filtering can sometimes help with filtering out noisy sentences that contain too many puncutations,
|
||||
it does not help in cases where the translations are potentially incorrect, disfluent, or incomplete. We use `bicleaner <https://github.com/bitextor/bicleaner>`__
|
||||
a tool to identify such sentences. It trains a classifier based on many features included pre-trained language model fluency, word
|
||||
alignment scores from a word-alignment model like `Giza++ <https://github.com/moses-smt/giza-pp>`__ etc. We use their available
|
||||
pre-trained models wherever possible and train models ourselves using their framework for remaining languages. The following script
|
||||
applies a pre-trained bicleaner model to the data and pick sentences that are clean with probability > 0.5.
|
||||
|
||||
.. code ::
|
||||
|
||||
awk '{print "-\t-"}' train.en \
|
||||
| paste -d "\t" - train.filter.en train.filter.es \
|
||||
| bicleaner-classify - - </path/to/bicleaner.yaml> > train.en-es.bicleaner.score
|
||||
|
||||
#. Data deduplication - We use `bifixer <https://github.com/bitextor/bifixer>`__ (which uses xxHash) to hash the source and target
|
||||
sentences based on which we remove duplicate entries from the file. You may want to do something similar to remove training examples
|
||||
that are in the test dataset.
|
||||
|
||||
.. code ::
|
||||
|
||||
cat train.en-es.bicleaner.score \
|
||||
| parallel -j 25 --pipe -k -l 30000 python bifixer.py --ignore-segmentation -q - - en es \
|
||||
> train.en-es.bifixer.score
|
||||
|
||||
awk -F awk -F "\t" '!seen[$6]++' train.en-es.bifixer.score > train.en-es.bifixer.dedup.score
|
||||
|
||||
#. Filter out data that bifixer assigns probability < 0.5 to.
|
||||
|
||||
.. code ::
|
||||
|
||||
awk -F "\t" '{ if ($5>0.5) {print $3}}' train.en-es.bifixer.dedup.score > train.cleaned.en
|
||||
awk -F "\t" '{ if ($5>0.5) {print $4}}' train.en-es.bifixer.dedup.score > train.cleaned.es
|
||||
|
||||
#. Punctuation Normalization - Punctuation, especially things like quotes can be written in different ways.
|
||||
It's often useful to normalize the way they appear in text. We use the moses punctuation normalizer on all languages except Chinese.
|
||||
|
||||
.. code ::
|
||||
|
||||
perl mosesdecoder/scripts/tokenizer/normalize-punctuation.perl -l es < train.cleaned.es > train.normalized.es
|
||||
perl mosesdecoder/scripts/tokenizer/normalize-punctuation.perl -l en < train.cleaned.en > train.normalized.en
|
||||
|
||||
For example:
|
||||
|
||||
.. code ::
|
||||
|
||||
Before - Aquí se encuentran joyerías como Tiffany`s entre negocios tradicionales suizos como la confitería Sprüngli.
|
||||
After - Aquí se encuentran joyerías como Tiffany's entre negocios tradicionales suizos como la confitería Sprüngli.
|
||||
|
||||
#. Tokenization and word segmentation for Chinese - Naturally written text often contains punctuation markers like commas, full-stops
|
||||
and apostrophes that are attached to words. Tokenization by just splitting a string on spaces will result in separate token IDs for
|
||||
very similar items like ``NeMo`` and ``NeMo.``. Tokenization splits punctuation from the word to create two separate tokens. In the
|
||||
previous example ``NeMo.`` becomes ``NeMo .`` which when split by space, results in two tokens and adressess the earlier problem.
|
||||
|
||||
For example:
|
||||
|
||||
.. code ::
|
||||
|
||||
Before - Especialmente porque se enfrentará "a Mathieu (Debuchy), Yohan (Cabaye) y Adil (Rami) ", recuerda.
|
||||
After - Especialmente porque se enfrentará " a Mathieu ( Debuchy ) , Yohan ( Cabaye ) y Adil ( Rami ) " , recuerda .
|
||||
|
||||
We use the Moses tokenizer for all languages except Chinese.
|
||||
|
||||
.. code ::
|
||||
|
||||
perl mosesdecoder/scripts/tokenizer/tokenizer.perl -l es -no-escape < train.normalized.es > train.tokenized.es
|
||||
perl mosesdecoder/scripts/tokenizer/tokenizer.perl -l en -no-escape < train.normalized.en > train.tokenized.en
|
||||
|
||||
For languages like Chinese where there is no explicit marker like spaces that separate words, we use `Jieba <https://github.com/fxsjy/jieba>`__ to segment a string into words that are space separated.
|
||||
|
||||
For example:
|
||||
|
||||
.. code ::
|
||||
|
||||
Before - 同时，卫生局认为有必要接种的其他人员，包括公共部门，卫生局将主动联络有关机构取得名单后由卫生中心安排接种。
|
||||
After - 同时 ， 卫生局 认为 有 必要 接种 的 其他 人员 ， 包括 公共部门 ， 卫生局 将 主动 联络 有关 机构 取得 名单 后 由 卫生 中心 安排 接种 。
|
||||
|
||||
Training a BPE Tokenization
|
||||
---------------------------
|
||||
|
||||
Byte-pair encoding (BPE) :cite:`nlp-machine_translation-sennrich2015neural` is a sub-word tokenization algorithm that is commonly used
|
||||
to reduce the large vocabulary size of datasets by splitting words into frequently occuring sub-words. Currently, mMachine translation
|
||||
only supports the `YouTokenToMe <https://github.com/VKCOM/YouTokenToMe>`__ BPE tokenizer. One can set the tokenization configuration
|
||||
as follows:
|
||||
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder_tokenizer,decoder_tokenizer}.tokenizer_name** | str | ``yttm`` | BPE library name. Only supports ``yttm`` for now. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder_tokenizer,decoder_tokenizer}.tokenizer_model** | str | ``null`` | Path to an existing YTTM BPE model. If ``null``, will train one from scratch on the provided data. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder_tokenizer,decoder_tokenizer}.vocab_size** | int | ``null`` | Desired vocabulary size after BPE tokenization. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder_tokenizer,decoder_tokenizer}.bpe_dropout** | float | ``null`` | BPE dropout probability. :cite:`nlp-machine_translation-provilkov2019bpe`. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder_tokenizer,decoder_tokenizer}.vocab_file** | str | ``null`` | Path to pre-computed vocab file if exists. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
| **model.shared_tokenizer** | bool | ``True`` | Whether to share the tokenizer between the encoder and decoder. |
|
||||
+-----------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
Applying BPE Tokenization, Batching, Bucketing and Padding
|
||||
----------------------------------------------------------
|
||||
|
||||
Given BPE tokenizers, and a cleaned parallel corpus, the following steps are applied to create a `TranslationDataset <https://github.com/NVIDIA/NeMo/blob/v1.0.2/nemo/collections/nlp/data/machine_translation/machine_translation_dataset.py#L64>`__ object.
|
||||
|
||||
#. Text to IDs - This performs subword tokenization with the BPE model on an input string and maps it to a sequence of tokens for the
|
||||
source and target text.
|
||||
|
||||
#. Bucketing - Sentences vary in length and when creating minibatches, we'd like sentences in them to have roughly the same length to
|
||||
minimize the number of ``<pad>`` tokens and to maximize computational efficiency. This step groups sentences roughly the same length
|
||||
into buckets.
|
||||
|
||||
#. Batching and padding - Creates minibatches with a maximum number of tokens specified by ``model.{train_ds,validation_ds,test_ds}.tokens_in_batch``
|
||||
from buckets and pads, so they can be packed into a tensor.
|
||||
|
||||
Datasets can be configured as follows:
|
||||
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.src_file_name** | str | ``null`` | Path to the source language file. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.tgt_file_name** | str | ``null`` | Path to the target language file. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.tokens_in_batch** | int | ``512`` | Maximum number of tokens per minibatch. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.clean** | bool | ``true`` | Whether to clean the dataset by discarding examples that are greater than ``max_seq_length``. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.max_seq_length** | int | ``512`` | Maximum sequence to be used with the ``clean`` argument above. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.shuffle** | bool | ``true`` | Whether to shuffle minibatches in the PyTorch DataLoader. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.num_samples** | int | ``-1`` | Number of samples to use. ``-1`` for the entire dataset. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.drop_last** | bool | ``false`` | Drop last minibatch if it is not of equal size to the others. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.pin_memory** | bool | ``false`` | Whether to pin memory in the PyTorch DataLoader. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.num_workers** | int | ``8`` | Number of workers for the PyTorch DataLoader. |
|
||||
+-------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
Tarred Datasets for Large Corpora
|
||||
---------------------------------
|
||||
|
||||
When training with ``DistributedDataParallel``, each process has its own copy of the dataset. For large datasets, this may not always
|
||||
fit in CPU memory. `Webdatasets <https://github.com/tmbdev/webdataset>`__ circumvents this problem by efficiently iterating over
|
||||
tar files stored on disk. Each tar file can contain hundreds to thousands of pickle files, each containing a single minibatch.
|
||||
|
||||
We recommend using this method when working with datasets with > 1 million sentence pairs.
|
||||
|
||||
Tarred datasets can be configured as follows:
|
||||
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.use_tarred_dataset** | bool | ``false`` | Whether to use tarred datasets. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.tar_files** | str | ``null`` | String specifying path to all tar files. Example with 100 tarfiles ``/path/to/tarfiles._OP_1..100_CL_.tar``. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.metadata_file** | str | ``null`` | Path to JSON metadata file that contains only a single entry for the total number of batches in the dataset. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.lines_per_dataset_fragment** | int | ``1000000`` | |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.num_batches_per_tarfile** | int | ``100`` | Maximum sequence to be used with the ``clean`` argument above. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.tar_shuffle_n** | int | ``100`` | Whether to cache IDs to avoid re-tokenizing data. This will be deprecated in favor of tarred datasets. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{train_ds,validation_ds,test_ds}.shard_strategy** | str | ``scatter`` | Whether to cache IDs in each of the nodes in multi-node training. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.preproc_out_dir** | str | ``null`` | Path to folder that contains processed tar files or directory where new tar files are written. |
|
||||
+-----------------------------------------------------------------------+-----------------+----------------+----------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
Tarred datasets can be created in two ways:
|
||||
|
||||
#. Using the Hydra config and `training script <https://github.com/NVIDIA/NeMo/blob/v1.0.2/examples/nlp/machine_translation/enc_dec_nmt.py>`__.
|
||||
|
||||
For example:
|
||||
|
||||
.. code ::
|
||||
|
||||
python examples/nlp/machine_translation/enc_dec_nmt.py \
|
||||
-cn aayn_base \
|
||||
do_training=false \
|
||||
model.preproc_out_dir=/path/to/preproc_dir \
|
||||
model.train_ds.use_tarred_dataset=true \
|
||||
model.train_ds.lines_per_dataset_fragment=1000000 \
|
||||
model.train_ds.num_batches_per_tarfile=200 \
|
||||
model.train_ds.src_file_name=train.tokenized.en \
|
||||
model.train_ds.tgt_file_name=train.tokenized.es \
|
||||
model.validation_ds.src_file_name=validation.tokenized.en \
|
||||
model.validation_ds.tgt_file_name=validation.tokenized.es \
|
||||
model.encoder_tokenizer.vocab_size=32000 \
|
||||
model.decoder_tokenizer.vocab_size=32000 \
|
||||
~model.test_ds \
|
||||
trainer.gpus=[0,1,2,3] \
|
||||
+trainer.fast_dev_run=true \
|
||||
exp_manager=null \
|
||||
|
||||
The above script processes the parallel tokenized text files into tarred datasets that are written to ``/path/to/preproc_dir``. Since
|
||||
``do_training`` is set to ``False``, the above script only creates tarred datasets and then exits. If ``do_training`` is set ``True``,
|
||||
then one of two things happen:
|
||||
|
||||
(a) If no tar files are present in ``model.preproc_out_dir``, the script first creates those files and then commences training.
|
||||
(b) If tar files are already present in ``model.preproc_out_dir``, the script starts training from the provided tar files.
|
||||
|
||||
#. Using a separate script without Hydra.
|
||||
|
||||
Tarred datasets for parallel corpora can also be created with a script that doesn't require specifying a configs via Hydra and
|
||||
just uses Python argparse.
|
||||
|
||||
For example:
|
||||
|
||||
.. code ::
|
||||
|
||||
python examples/nlp/machine_translation/create_tarred_parallel_dataset.py \
|
||||
--shared_tokenizer \
|
||||
--clean \
|
||||
--bpe_dropout 0.1 \
|
||||
--src_fname train.tokenized.en \
|
||||
--tgt_fname train.tokenized.es \
|
||||
--out_dir /path/to/preproc_dir \
|
||||
--vocab_size 32000 \
|
||||
--max_seq_length 512 \
|
||||
--min_seq_length 1 \
|
||||
--tokens_in_batch 8192 \
|
||||
--lines_per_dataset_fragment 1000000 \
|
||||
--num_batches_per_tarfile 200
|
||||
|
||||
You can then set `model.preproc_out_dir=/path/to/preproc_dir` and `model.train_ds.use_tarred_dataset=true` to train with this data.
|
||||
|
||||
Model Configuration and Training
|
||||
--------------------------------
|
||||
|
||||
The overall model consists of an encoder, decoder, and classification head. Encoders and decoders have the following configuration
|
||||
options:
|
||||
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.max_sequence_length** | int | ``512`` | Maximum sequence length of positional encodings. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.embedding_dropout** | float | ``0.1`` | Path to JSON metadata file that contains only a single entry for the total number of batches in the dataset. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.learn_positional_encodings** | bool | ``false`` | If ``True``, this is a regular learnable embedding layer. If ``False``, fixes position encodings to sinusoidal. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.hidden_size** | int | ``512`` | Size of the transformer hidden states. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.num_layers** | int | ``6`` | Number of transformer layers. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.inner_size** | int | ``2048`` | Size of the hidden states within the feedforward layers. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.num_attention_heads** | int | ``8`` | Number of attention heads. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.ffn_dropout** | float | ``0.1`` | Dropout probability within the feedforward layers. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.attn_score_dropout** | float | ``0.1`` | Dropout probability of the attention scores before softmax normalization. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.attn_layer_dropout** | float | ``0.1`` | Dropout probability of the attention query, key, and value projection activations. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.hidden_act** | str | ``relu`` | Activation function throughout the network. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.mask_future** | bool | ``false``, ``true`` | Whether to mask future timesteps for attention. Defaults to ``True`` for decoder and ``False`` for encoder. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
| **model.{encoder,decoder}.pre_ln** | bool | ``false`` | Whether to apply layer-normalization before (``true``) or after (``false``) a sub-layer. |
|
||||
+-------------------------------------------------------------------+-----------------+-----------------------+-----------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
Our pre-trained models are optimized with Adam, with a maximum learning of 0.0004, beta of (0.9, 0.98), an inverse square root learning
|
||||
rate schedule from :cite:`nlp-machine_translation-vaswani2017attention`. The **model.optim** section sets the optimization parameters.
|
||||
|
||||
The following script creates tarred datasets based on the provided parallel corpus and train a model based on the ``base`` configuration
|
||||
from :cite:`nlp-machine_translation-vaswani2017attention`.
|
||||
|
||||
.. code ::
|
||||
|
||||
python examples/nlp/machine_translation/enc_dec_nmt.py \
|
||||
-cn aayn_base \
|
||||
do_training=true \
|
||||
trainer.gpus=8 \
|
||||
~trainer.max_epochs \
|
||||
+trainer.max_steps=100000 \
|
||||
+trainer.val_check_interval=1000 \
|
||||
+exp_manager.exp_dir=/path/to/store/results \
|
||||
+exp_manager.create_checkpoint_callback=True \
|
||||
+exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU \
|
||||
+exp_manager.checkpoint_callback_params.mode=max \
|
||||
+exp_manager.checkpoint_callback_params.save_top_k=5 \
|
||||
model.preproc_out_dir=/path/to/preproc_dir \
|
||||
model.train_ds.use_tarred_dataset=true \
|
||||
model.train_ds.lines_per_dataset_fragment=1000000 \
|
||||
model.train_ds.num_batches_per_tarfile=200 \
|
||||
model.train_ds.src_file_name=train.tokenized.en \
|
||||
model.train_ds.tgt_file_name=train.tokenized.es \
|
||||
model.validation_ds.src_file_name=validation.tokenized.en \
|
||||
model.validation_ds.tgt_file_name=validation.tokenized.es \
|
||||
model.encoder_tokenizer.vocab_size=32000 \
|
||||
model.decoder_tokenizer.vocab_size=32000 \
|
||||
~model.test_ds \
|
||||
|
||||
The trainer keeps track of the sacreBLEU score :cite:`nlp-machine_translation-post2018call` on the provided validation set and saves
|
||||
the checkpoints that had the top 5 (by default) sacreBLEU scores.
|
||||
|
||||
At the end of training, a ``.nemo`` file is written to the result directory which allows to run inference on a test set.
|
||||
|
||||
Multi-Validation
|
||||
----------------
|
||||
|
||||
To run validation on multiple datasets, specify ``validation_ds.src_file_name`` and ``validation_ds.tgt_file_name`` with a list of file paths:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
model.validation_ds.src_file_name=[/data/wmt13-en-de.src,/data/wmt14-en-de.src] \
|
||||
model.validation_ds.tgt_file_name=[/data/wmt13-en-de.ref,/data/wmt14-en-de.ref] \
|
||||
|
||||
When using ``val_loss`` or ``val_sacreBLEU`` for the ``exp_manager.checkpoint_callback_params.monitor``
|
||||
then the 0th indexed dataset will be used as the monitor.
|
||||
|
||||
To use other indexes, append the index:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
exp_manager.checkpoint_callback_params.monitor=val_sacreBLEU_dl_index_1
|
||||
|
||||
Multiple test datasets work exactly the same way as validation datasets, simply replace ``validation_ds`` by ``test_ds`` in the above examples.
|
||||
|
||||
Model Inference
|
||||
---------------
|
||||
|
||||
To generate translations on a test set and compute sacreBLEU scores, run the inference script:
|
||||
|
||||
.. code ::
|
||||
|
||||
python examples/nlp/machine_translation/nmt_transformer_infer.py \
|
||||
--model /path/to/model.nemo \
|
||||
--srctext test.en \
|
||||
--tgtout test.en-es.translations \
|
||||
--batch_size 128 \
|
||||
--source_lang en \
|
||||
--target_lang es
|
||||
|
||||
The ``--srctext`` file must be provided before tokenization and normalization. The resulting ``--tgtout`` file is detokenized and
|
||||
can be used to compute sacreBLEU scores.
|
||||
|
||||
.. code ::
|
||||
|
||||
cat test.en-es.translations | sacrebleu test.es
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: nlp-machine_translation
|
||||
:keyprefix: nlp-machine_translation-
|
||||
@@ -0,0 +1,216 @@
|
||||
.. _megatron_finetuning:
|
||||
|
||||
Megatron-LM for Downstream Tasks
|
||||
================================
|
||||
|
||||
Megatron-LM :cite:`nlp-megatron-shoeybi2019megatron` is a large, powerful transformer developed by the Applied Deep Learning Research
|
||||
team at NVIDIA. Unlike BERT, the position of the layer normalization and the residual connection in the model architecture (similar to
|
||||
GPT-2 architucture) are swapped, which allows the models to continue to improve as they were scaled up. This model reaches higher
|
||||
scores compared to BERT on a range of Natural Language Processing (NLP) tasks. More details on efficient, model-parallel and multi-node
|
||||
pre-training of GPT and BERT using mixed precision can be found in the `Megatron-LM GitHub repo <https://github.com/NVIDIA/Megatron-LM>`_.
|
||||
|
||||
|
||||
Fine-tuning
|
||||
-----------
|
||||
|
||||
Pre-trained Megatron-LM (BERT) can be used for most of the NLP downstream tasks from `NeMo/examples/nlp <https://github.com/NVIDIA/NeMo/tree/master/examples/nlp>`_.
|
||||
Specify the ``model.language_model.pretrained_model_name`` parameter:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
model.language_model.pretrained_model_name=megatron-bert-345m-uncased
|
||||
|
||||
Available pre-trained Megatron-LM models:
|
||||
|
||||
- ``megatron-bert-345m-cased``
|
||||
- ``megatron-bert-345m-uncased``
|
||||
- ``biomegatron-bert-345m-uncased``
|
||||
- ``biomegatron-bert-345m-cased``
|
||||
|
||||
For example, to fine-tune SQuAD v1.1 with Megatron-LM, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python question_answering_squad.py \
|
||||
model.train_ds.file=<TRAIN_JSON_FILE> \
|
||||
model.validation_ds=<VAL_JSON_FILE> \
|
||||
model.language_model.pretrained_model_name=megatron-bert-345m-uncased
|
||||
|
||||
If you have a different checkpoint or model configuration (pre-trained with `Megatron-LM GitHub repo <https://github.com/NVIDIA/Megatron-LM>`_),
|
||||
use ``model.language_model.pretrained_model_name=megatron-bert-uncased`` or ``model.language_model.pretrained_model_name=megatron-bert-cased``
|
||||
and specify ``--bert_config`` and ``--bert_checkpoint`` for your model.
|
||||
|
||||
.. note::
|
||||
Megatron-LM has its own set of training arguments (including tokenizer) that are ignored during fine-tuning in NeMo. Use downstream
|
||||
task training scripts for all NeMo supported arguments.
|
||||
|
||||
BioMegatron
|
||||
-----------
|
||||
|
||||
BioMegatron has the same network architecture as the Megatron-LM, but is pretrained on a different dataset - `PubMed <https://catalog.data.gov/dataset/pubmed>`_,
|
||||
a large biomedical text corpus, which achieves better performance in biomedical downstream tasks than the original Megatron-LM.
|
||||
|
||||
Examples of using BioMegatron on biomedical downstream tasks can be found at (can be executed with `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_):
|
||||
`NeMo/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb>`__ and `NeMo/tutorials/nlp/Token_Classification-BioMegatron.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Token_Classification-BioMegatron.ipynb>`__.
|
||||
|
||||
Model Parallelism
|
||||
-----------------
|
||||
|
||||
`Megatron-LM <https://github.com/NVIDIA/Megatron-LM>`_ is a highly optimized and efficient library for training large language models.
|
||||
With Megatron model parallelism, language models can be trained with billions of weights and then used in NeMo for downstream tasks.
|
||||
|
||||
NeMo handles pretrained model parallel checkpoints from Megatron-LM automatically and model parallel models in NeMo have the all
|
||||
the same features as other NeMo Models.
|
||||
|
||||
.. note::
|
||||
|
||||
Currently, NeMo only supports tensor model parallelism.
|
||||
|
||||
Training
|
||||
^^^^^^^^
|
||||
|
||||
All of the necessary logic to train model parallel models in NeMo with PyTorch Lightning is contained in the ``NLPDDPPlugin``.
|
||||
The ``NLPDDPPlugin`` subclasses the PyTorch Lightning training type plugin ``DDPPlugin``.
|
||||
See `plugins <https://pytorch-lightning.readthedocs.io/en/latest/extensions/plugins.html>`_ for more information on PyTorch Lightning Plugins.
|
||||
|
||||
To enable model parallel training in NeMo:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
trainer = Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer)
|
||||
|
||||
Megatron-LM checkpoints have a specific format. One checkpoint is saved for each model parallel rank:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
iter_0080000/
|
||||
├── mp_rank_00
|
||||
│ └── model_optim_rng.pt
|
||||
└── mp_rank_01
|
||||
└── model_optim_rng.pt
|
||||
|
||||
|
||||
To start fine-tuning from a Megatron-LM checkpoint, simply pass the path to the Megatron-LM checkpoint
|
||||
via the language model config:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
model.language_model.lm_checkpoint=/raid/megatron/bert/iter_0080000 \
|
||||
|
||||
We also need to input the model configuration. This can be done via json:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"hidden-size": 1024,
|
||||
"num-attention-heads": 16,
|
||||
"num-layers": 24,
|
||||
"max-seq-length": 512
|
||||
}
|
||||
|
||||
And input via command line:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
model.language_model.config_file=/raid/data/megatron/bert/config.json \
|
||||
|
||||
Or the model configuration can be input via YAML:
|
||||
|
||||
.. code-block:: YAML
|
||||
|
||||
model:
|
||||
language_model:
|
||||
config:
|
||||
hidden_size: 1024
|
||||
num_attention_heads: 16
|
||||
num_layers: 24
|
||||
max_position_embeddings: 512
|
||||
|
||||
Additionally, Megatron-LM requires a vocab file:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
model.tokenizer.vocab_file=/path/to/vocab.txt
|
||||
|
||||
If using the Megatron-LM default tokenizer for training BERT the vocab file can be omitted:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# uncased model
|
||||
model.tokenizer.tokenizer_name=megatron-bert-uncased
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# cased model
|
||||
model.tokenizer.tokenizer_name=megatron-bert-uncased
|
||||
|
||||
Auto-Resume
|
||||
^^^^^^^^^^^
|
||||
|
||||
Resuming training with NeMo experiment manager and PyTorch Lightning works exactly the same as other NeMo models.
|
||||
While training with PTL, model parallel checkpoint will be saved and loaded properly.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
checkpoints/
|
||||
├── mp_rank_00
|
||||
│ ├── mp_autoresume-last.ckpt
|
||||
│ ├── mp_autoresume---val_loss=0.35-epoch=0.ckpt
|
||||
│ ├── mp_autoresume---val_loss=0.38-epoch=1.ckpt
|
||||
│ └── mp_autoresume---val_loss=0.39-epoch=2.ckpt
|
||||
└── mp_rank_01
|
||||
├── mp_autoresume-last.ckpt
|
||||
├── mp_autoresume---val_loss=0.35-epoch=0.ckpt
|
||||
├── mp_autoresume---val_loss=0.38-epoch=1.ckpt
|
||||
└── mp_autoresume---val_loss=0.39-epoch=2.ckpt
|
||||
|
||||
Save and Restore
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
Model parallel .nemo files behave the same as all other .nemo files. Calling ``.save_to`` will save
|
||||
a checkpoint for each model parallel rank inside the .nemo file:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
text_class_350m
|
||||
├── megatron-bert-uncased_encoder_config.json
|
||||
├── megatron_checkpoint_version.json
|
||||
├── model_config.yaml
|
||||
├── mp_rank_00
|
||||
│ └── model_weights.ckpt
|
||||
├── mp_rank_01
|
||||
│ └── model_weights.ckpt
|
||||
├── tokenizer_vocab_dict.json
|
||||
└── tokenizer.vocab_file
|
||||
|
||||
When restoring a model parallel .nemo file, we must pass in the ``Trainer`` as model parallel requires DDP:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
model = TokenClassificationModel.restore_from(cfg.pretrained_model, trainer=trainer)
|
||||
|
||||
Evaluation
|
||||
^^^^^^^^^^
|
||||
|
||||
Since model parallel models always require more than one GPU, the ``Trainer`` is needed for evaluation:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
trainer = pl.Trainer(plugins=[NLPDDPPlugin()], **cfg.trainer)
|
||||
|
||||
model = TextClassificationModel.restore_from(cfg.model.nemo_path, trainer=trainer)
|
||||
model.setup_test_data(test_data_config=cfg.model.test_ds)
|
||||
|
||||
trainer.test(model=model, ckpt_path=None)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-MEGATRON
|
||||
:keyprefix: nlp-megatron-
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
.. _nlp_models:
|
||||
|
||||
Models
|
||||
======
|
||||
|
||||
NeMo's NLP collection supports the following models:
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
punctuation_and_capitalization
|
||||
token_classification
|
||||
joint_intent_slot
|
||||
text_classification
|
||||
bert_pretraining
|
||||
language_modeling
|
||||
question_answering
|
||||
sgd_qa
|
||||
glue_benchmark
|
||||
information_retrieval
|
||||
nlp_model
|
||||
machine_translation
|
||||
@@ -0,0 +1,109 @@
|
||||
@article{devlin2018bert,
|
||||
title={Bert: Pre-training of deep bidirectional transformers for language understanding},
|
||||
author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
|
||||
journal={arXiv preprint arXiv:1810.04805},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@article{shoeybi2019megatron,
|
||||
title={Megatron-lm: Training multi-billion parameter language models using model parallelism},
|
||||
author={Shoeybi, Mohammad and Patwary, Mostofa and Puri, Raul and LeGresley, Patrick and Casper, Jared and Catanzaro, Bryan},
|
||||
journal={arXiv preprint arXiv:1909.08053},
|
||||
year={2019}
|
||||
}
|
||||
|
||||
@InProceedings{maas2011,
|
||||
author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher},
|
||||
title = {Learning Word Vectors for Sentiment Analysis},
|
||||
booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies},
|
||||
month = {June},
|
||||
year = {2011},
|
||||
address = {Portland, Oregon, USA},
|
||||
publisher = {Association for Computational Linguistics},
|
||||
pages = {142--150},
|
||||
url = {http://www.aclweb.org/anthology/P11-1015}
|
||||
}
|
||||
|
||||
@inproceedings{socher2013,
|
||||
title = "Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank",
|
||||
author = "Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D. and Ng, Andrew and Potts, Christopher",
|
||||
booktitle = "Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing",
|
||||
month = oct,
|
||||
year = "2013",
|
||||
address = "Seattle, Washington, USA",
|
||||
publisher = "Association for Computational Linguistics",
|
||||
url = "https://www.aclweb.org/anthology/D13-1170",
|
||||
pages = "1631--1642",
|
||||
}
|
||||
|
||||
@article{lim2018chemical,
|
||||
title={Chemical--gene relation extraction using recursive neural network},
|
||||
author={Lim, Sangrak and Kang, Jaewoo},
|
||||
journal={Database},
|
||||
volume={2018},
|
||||
year={2018},
|
||||
publisher={Oxford Academic}
|
||||
}
|
||||
|
||||
@inproceedings{li2007scalable,
|
||||
title={Scalable term selection for text categorization},
|
||||
author={Li, Jingyang and Sun, Maosong},
|
||||
booktitle={Proceedings of the 2007 Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP-CoNLL)},
|
||||
pages={774--782},
|
||||
year={2007}
|
||||
}
|
||||
|
||||
@misc{lee2019biobert,
|
||||
title={BioBERT: a pre-trained biomedical language representation model for biomedical text mining},
|
||||
author={Jinhyuk Lee and Wonjin Yoon and Sungdong Kim and Donghyeon Kim and Sunkyu Kim and Chan Ho So and Jaewoo Kang},
|
||||
year={2019},
|
||||
eprint={1901.08746},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
|
||||
@misc{shin2020biomegatron,
|
||||
title={BioMegatron: Larger Biomedical Domain Language Model},
|
||||
author={Hoo-Chang Shin and Yang Zhang and Evelina Bakhturina and Raul Puri and Mostofa Patwary and Mohammad Shoeybi and Raghav Mani},
|
||||
year={2020},
|
||||
eprint={2010.06060},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
|
||||
@article{vaswani2017attention,
|
||||
title={Attention is all you need},
|
||||
author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, Lukasz and Polosukhin, Illia},
|
||||
journal={arXiv preprint arXiv:1706.03762},
|
||||
year={2017}
|
||||
}
|
||||
|
||||
@article{sennrich2015neural,
|
||||
title={Neural machine translation of rare words with subword units},
|
||||
author={Sennrich, Rico and Haddow, Barry and Birch, Alexandra},
|
||||
journal={arXiv preprint arXiv:1508.07909},
|
||||
year={2015}
|
||||
}
|
||||
|
||||
@article{provilkov2019bpe,
|
||||
title={Bpe-dropout: Simple and effective subword regularization},
|
||||
author={Provilkov, Ivan and Emelianenko, Dmitrii and Voita, Elena},
|
||||
journal={arXiv preprint arXiv:1910.13267},
|
||||
year={2019}
|
||||
}
|
||||
|
||||
@article{post2018call,
|
||||
title={A call for clarity in reporting BLEU scores},
|
||||
author={Post, Matt},
|
||||
journal={arXiv preprint arXiv:1804.08771},
|
||||
year={2018}
|
||||
}
|
||||
|
||||
@misc{zhang2021sgdqa,
|
||||
title={SGD-QA: Fast Schema-Guided Dialogue State Tracking for Unseen Services},
|
||||
author={Yang Zhang and Vahid Noroozi and Evelina Bakhturina and Boris Ginsburg},
|
||||
year={2021},
|
||||
eprint={2105.08049},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
.. _nlp_model:
|
||||
|
||||
Model NLP
|
||||
=========
|
||||
|
||||
The config file for NLP models contain three main sections:
|
||||
|
||||
- ``trainer``: contains the configs for PTL training. For more information, refer to :doc:`../core/core` and `PTL Trainer class API <https://pytorch-lightning.readthedocs.io/en/stable/common/trainer.html#trainer-class-api>'.
|
||||
- ``exp_manager``: the configs of the experiment manager. For more information, refer to :doc:`../core/core`.
|
||||
- ``model``: contains the configs of the datasets, model architecture, tokenizer, optimizer, scheduler, etc.
|
||||
|
||||
The following sub-sections of the model section are shared among most of the NLP models.
|
||||
|
||||
- ``tokenizer``: specifies the tokenizer
|
||||
- ``language_model``: specifies the underlying model to be used as the encoder
|
||||
- ``optim``: the configs of the optimizer and scheduler :doc:`../core/core`
|
||||
|
||||
The ``tokenizer`` and ``language_model`` sections have the following parameters:
|
||||
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.tokenizer.tokenizer_name** | string | Tokenizer name will be filled automatically based on ``model.language_model.pretrained_model_name``. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.tokenizer.vocab_file** | string | Path to tokenizer vocabulary. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.tokenizer.tokenizer_model** | string | Path to tokenizer model (only for sentencepiece tokenizer). |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.language_model.pretrained_model_name** | string | Pre-trained language model name, for example: ``bert-base-cased`` or ``bert-base-uncased``. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.language_model.lm_checkpoint** | string | Path to the pre-trained language model checkpoint. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.language_model.config_file** | string | Path to the pre-trained language model config file. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.language_model.config** | dictionary | Config of the pre-trained language model. |
|
||||
+------------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
The parameter **model.language_model.pretrained_model_name** can be one of the following:
|
||||
|
||||
- ``megatron-bert-345m-uncased``, ``megatron-bert-345m-cased``, ``biomegatron-bert-345m-uncased``, ``biomegatron-bert-345m-cased``, ``bert-base-uncased``, ``bert-large-uncased``, ``bert-base-cased``, ``bert-large-cased``
|
||||
- ``distilbert-base-uncased``, ``distilbert-base-cased``
|
||||
- ``roberta-base``, ``roberta-large``, ``distilroberta-base``
|
||||
- ``albert-base-v1``, ``albert-large-v1``, ``albert-xlarge-v1``, ``albert-xxlarge-v1``, ``albert-base-v2``, ``albert-large-v2``, ``albert-xlarge-v2``, ``albert-xxlarge-v2``
|
||||
+300
@@ -0,0 +1,300 @@
|
||||
.. _punctuation_and_capitalization:
|
||||
|
||||
Punctuation and Capitalization Model
|
||||
====================================
|
||||
|
||||
Automatic Speech Recognition (ASR) systems typically generate text with no punctuation and capitalization of the words.
|
||||
There are two issues with non-punctuated ASR output:
|
||||
|
||||
- it could be difficult to read and understand
|
||||
- models for some downstream tasks, such as named entity recognition, machine translation, or text-to-speech, are usually trained
|
||||
on punctuated datasets and using raw ASR output as the input to these models could deteriorate their performance
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.collections.nlp.models import PunctuationCapitalizationModel
|
||||
|
||||
# to get the list of pre-trained models
|
||||
PunctuationCapitalizationModel.list_available_models()
|
||||
|
||||
# Download and load the pre-trained BERT-based model
|
||||
model = PunctuationCapitalizationModel.from_pretrained("punctuation_en_bert")
|
||||
|
||||
# try the model on a few examples
|
||||
model.add_punctuation_capitalization(['how are you', 'great how about you'])
|
||||
|
||||
Model Description
|
||||
-----------------
|
||||
|
||||
For each word in the input text, the Punctuation and Capitalization model:
|
||||
|
||||
- predicts a punctuation mark that should follow the word (if any). By default, the model supports commas, periods, and question marks.
|
||||
- predicts if the word should be capitalized or not
|
||||
|
||||
In the Punctuation and Capitalization model, we are jointly training two token-level classifiers on top of a pre-trained
|
||||
language model, such as `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ :cite:`nlp-punct-devlin2018bert`.
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend you try this model in a Jupyter notebook (run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.): `NeMo/tutorials/nlp/Punctuation_and_Capitalization.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Punctuation_and_Capitalization.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator).
|
||||
|
||||
An example script on how to train the model can be found at: `NeMo/examples/nlp/token_classification/punctuation_capitalization_train.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/punctuation_capitalization_train.py>`__.
|
||||
|
||||
An example script on how to run evaluation and inference can be found at: `NeMo/examples/nlp/token_classification/punctuation_capitalization_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/punctuation_capitalization_evaluate.py>`__.
|
||||
|
||||
The default configuration file for the model can be found at: `NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml>`__.
|
||||
|
||||
.. _raw_data_format_punct:
|
||||
|
||||
Raw Data Format
|
||||
---------------
|
||||
|
||||
The Punctuation and Capitalization model can work with any text dataset, although it is recommended to balance the data, especially
|
||||
for the punctuation task. Before pre-processing the data to the format expected by the model, the data should be split into ``train.txt``
|
||||
and ``dev.txt`` (and optionally ``test.txt``). Each line in the ``train.txt/dev.txt/test.txt`` should represent one or more full
|
||||
and/or truncated sentences.
|
||||
|
||||
Example of the ``train.txt``/``dev.txt`` file:
|
||||
|
||||
.. code::
|
||||
|
||||
When is the next flight to New York?
|
||||
The next flight is ...
|
||||
....
|
||||
|
||||
|
||||
The ``source_data_dir`` structure should look similar to the following:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--sourced_data_dir
|
||||
|-- dev.txt
|
||||
|-- train.txt
|
||||
|
||||
NeMo Data Format
|
||||
----------------
|
||||
|
||||
The Punctuation and Capitalization model expects the data in the following format:
|
||||
|
||||
The training and evaluation data is divided into 2 files:
|
||||
- ``text.txt``
|
||||
- ``labels.txt``
|
||||
|
||||
Each line of the ``text.txt`` file contains text sequences, where words are separated with spaces.
|
||||
|
||||
[WORD] [SPACE] [WORD] [SPACE] [WORD], for example:
|
||||
|
||||
::
|
||||
|
||||
when is the next flight to new york
|
||||
the next flight is ...
|
||||
...
|
||||
|
||||
The ``labels.txt`` file contains corresponding labels for each word in ``text.txt``, the labels are separated with spaces.
|
||||
Each label in ``labels.txt`` file consists of 2 symbols:
|
||||
|
||||
- the first symbol of the label indicates what punctuation mark should follow the word (where ``O`` means no punctuation needed)
|
||||
- the second symbol determines if a word needs to be capitalized or not (where ``U`` indicates that the word should be upper cased,
|
||||
and ``O`` - no capitalization needed)
|
||||
|
||||
By default, the following punctuation marks are considered: commas, periods, and question marks; the remaining punctuation marks were
|
||||
removed from the data. This can be changed by introducing new labels in the ``labels.txt`` files.
|
||||
|
||||
Each line of the ``labels.txt`` should follow the format: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]`` (for ``labels.txt``). For example,
|
||||
labels for the above ``text.txt`` file should be:
|
||||
|
||||
::
|
||||
|
||||
OU OO OO OO OO OO OU ?U
|
||||
OU OO OO OO ...
|
||||
...
|
||||
|
||||
The complete list of all possible labels used in this tutorial are:
|
||||
|
||||
- ``OO``
|
||||
- ``O``
|
||||
- ``.O``
|
||||
- ``?O``
|
||||
- ``OU``
|
||||
- <blank space>
|
||||
- ``U``
|
||||
- ``.U``
|
||||
- ``?U``
|
||||
|
||||
Converting Raw Data to NeMo Format
|
||||
----------------------------------
|
||||
|
||||
To pre-process the raw text data, stored under :code:`sourced_data_dir` (see the :ref:`raw_data_format_punct`
|
||||
section), run the following command:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/token_classification/data/prepare_data_for_punctuation_capitalization.py \
|
||||
-s <PATH_TO_THE_SOURCE_FILE>
|
||||
-o <PATH_TO_THE_OUTPUT_DIRECTORY>
|
||||
|
||||
|
||||
Required Argument for Dataset Conversion
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`-s` or :code:`--source_file`: path to the raw file
|
||||
- :code:`-o` or :code:`--output_dir` - path to the directory to store the converted files
|
||||
|
||||
After the conversion, the :code:`output_dir` should contain :code:`labels_*.txt` and :code:`text_*.txt` files. The default names
|
||||
for the training and evaluation in the :code:`conf/punctuation_capitalization_config.yaml` are the following:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--output_dir
|
||||
|-- labels_dev.txt
|
||||
|-- labels_train.txt
|
||||
|-- text_dev.txt
|
||||
|-- text_train.txt
|
||||
|
||||
Training Punctuation and Capitalization Model
|
||||
---------------------------------------------
|
||||
|
||||
The language model is initialized with the pre-trained model from `HuggingFace Transformers <https://github.com/huggingface/transformers>`__,
|
||||
unless the user provides a pre-trained checkpoint for the language model. Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml>`__.
|
||||
|
||||
The specification is roughly grouped into the following categories:
|
||||
|
||||
- Parameters that describe the training process: **trainer**
|
||||
- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds**
|
||||
- Parameters that describe the model: **model**
|
||||
|
||||
More details about parameters in the config file can be found below and in the `model's config file <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/conf/punctuation_capitalization_config.yaml>`__:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **pretrained_model** | string | Path to the pre-trained model ``.nemo`` file or pre-trained model name. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.data_dir** | string | Path to the data converted to the specified above format. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.punct_head.punct_num_fc_layers** | integer | Number of fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.punct_head.fc_dropout** | float | Activation to use between fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.punct_head.activation** | string | Dropout to apply to the input hidden states. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.punct_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.capit_head.punct_num_fc_layers** | integer | Number of fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.capit_head.fc_dropout** | float | Dropout to apply to the input hidden states. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.capit_head.activation** | string | Activation function to use between fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.capit_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.text_file** | string | Name of the text training file located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.labels_file** | string | Name of the labels training file located at ``data_dir``, such as ``labels_train.txt``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.text_file** | string | Name of the text file for evaluation, located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.labels_file** | string | Name of the labels dev file located at ``data_dir``, such as ``labels_dev.txt``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.num_samples** | integer | Number of samples to use from the dev set, ``-1`` - to use all. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
For more information, refer to the :ref:`nlp_model` section.
|
||||
|
||||
To train the model from scratch, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/token_classification/punctuation_and_capitalization_train.py \
|
||||
model.dataset.data_dir=<PATH/TO/DATA_DIR> \
|
||||
trainer.gpus=[0,1] \
|
||||
optim.name=adam \
|
||||
optim.lr=0.0001
|
||||
|
||||
The above command will start model training on GPUs 0 and 1 with Adam optimizer and learning rate of 0.0001; and the trained model is
|
||||
stored in the ``nemo_experiments/Punctuation_and_Capitalization`` folder.
|
||||
|
||||
To train from the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/token_classification/punctuation_and_capitalization_train.py \
|
||||
model.dataset.data_dir=<PATH/TO/DATA_DIR> \
|
||||
pretrained_model=<PATH/TO/SAVE/.nemo>
|
||||
|
||||
|
||||
Required Arguments for Training
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
* :code:`model.dataset.data_dir`: Path to the `data_dir` with the pre-processed data files.
|
||||
|
||||
|
||||
.. note::
|
||||
|
||||
All parameters defined in the configuration file can be changed with command arguments. For example, the sample config file
|
||||
mentioned above has :code:`validation_ds.batch_size` set to ``64``. However, if you see that the GPU utilization can be
|
||||
optimized further by using a larger batch size, you may override to the desired value by adding the field :code:`validation_ds.batch_size=128`
|
||||
over the command-line. You can repeat this with any of the parameters defined in the sample configuration file.
|
||||
|
||||
Inference
|
||||
---------
|
||||
|
||||
An example script on how to run inference on a few examples, can be found at `examples/nlp/token_classification/punctuation_capitalization_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/punctuation_capitalization_evaluate.py>`_.
|
||||
|
||||
To start inference with a pre-trained model on a few examples, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python punctuation_capitalization_evaluate.py \
|
||||
pretrained_model=<PRETRAINED_MODEL>
|
||||
|
||||
|
||||
Model Evaluation
|
||||
----------------
|
||||
|
||||
An example script on how to evaluate the pre-trained model, can be found at `examples/nlp/token_classification/punctuation_capitalization_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/punctuation_capitalization_evaluate.py>`_.
|
||||
|
||||
To start evaluation of the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python punctuation_capitalization_evaluate.py \
|
||||
model.dataset.data_dir=<PATH/TO/DATA/DIR> \
|
||||
pretrained_model=punctuation_en_bert \
|
||||
model.test_ds.text_file=<text_dev.txt> \
|
||||
model.test_ds.labels_file=<labels_dev.txt>
|
||||
|
||||
|
||||
Required Arguments
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`pretrained_model`: pretrained Punctuation and Capitalization model from ``list_available_models()`` or path to a ``.nemo``
|
||||
file. For example: ``punctuation_en_bert`` or ``your_model.nemo``.
|
||||
- :code:`model.dataset.data_dir`: path to the directory that containes :code:`model.test_ds.text_file` and :code:`model.test_ds.labels_file`
|
||||
|
||||
During evaluation of the :code:`test_ds`, the script generates two classification reports: one for capitalization task and another
|
||||
one for punctuation task. This classification reports include the following metrics:
|
||||
|
||||
- :code:`Precision`
|
||||
- :code:`Recall`
|
||||
- :code:`F1`
|
||||
|
||||
More details about these metrics can be found `here <https://en.wikipedia.org/wiki/Precision_and_recall>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-PUNCT
|
||||
:keyprefix: nlp-punct-
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
.. _question_answering:
|
||||
|
||||
Question Answering model
|
||||
========================
|
||||
|
||||
With Question Answering, or Reading Comprehension, given a question and a passage of content (context) that may contain an answer for
|
||||
the question, the model predicts the span within the text with a start and end position indicating the answer to the question. For
|
||||
datasets like SQuAD 2.0, this model supports cases when the answer is not contained in the content.
|
||||
|
||||
For every word in the context of a given question, the model is trained to predict:
|
||||
|
||||
- The likelihood this word is the start of the span.
|
||||
- The likelihood this word is the end of the span.
|
||||
|
||||
The model chooses the start and end words with maximal probabilities. When the content does not contain the answer, we would like the
|
||||
start and end span to be set for the first token.
|
||||
|
||||
A pretrained BERT encoder with two span prediction heads is used for the prediction start and the end position of the answer. The span
|
||||
predictions are token classifiers consisting of a single linear layer.
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.collections.nlp.models import QAModel
|
||||
|
||||
# to get the list of pre-trained models
|
||||
QAModel.list_available_models()
|
||||
|
||||
# Download and load the pre-trained BERT-based model
|
||||
model = QAModel.from_pretrained("qa_squadv1.1_bertbase")
|
||||
|
||||
# try the model on a few examples
|
||||
model.inference(test_file)
|
||||
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend you try Question Answering model in a Jupyter notebook (can run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.):
|
||||
`NeMo/tutorials/nlp/Question_Answering_Squad.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Question_Answering_Squad.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator).
|
||||
|
||||
An example script on how to train and evaluate the model can be found here: `NeMo/examples/nlp/question_answering/question_answering_squad.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/question_answering/question_answering_squad.py>`__.
|
||||
|
||||
The default configuration file for the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/question_answering/conf/question_answering_squad_config.yaml>`__.
|
||||
|
||||
|
||||
|
||||
Available models
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. list-table:: *Pretrained Models*
|
||||
:widths: 5 10
|
||||
:header-rows: 1
|
||||
|
||||
* - Model
|
||||
- Pretrained Checkpoint
|
||||
* - qa_squadv1.1_bertbase
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_bertbase
|
||||
* - qa_squadv2.0_bertbase
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_bertbase
|
||||
* - qa_squadv1.1_bertlarge
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_bertlarge
|
||||
* - qa_squadv2.0_bertlarge
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_bertlarge
|
||||
* - qa_squadv1.1_megatron_cased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_megatron_cased
|
||||
* - qa_squadv2.0_megatron_cased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_megatron_cased
|
||||
* - qa_squadv1.1_megatron_uncased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv1_1_megatron_uncased
|
||||
* - qa_squadv2.0_megatron_uncased
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:qa_squadv2_0_megatron_uncased
|
||||
|
||||
|
||||
.. _dataset_question_answering:
|
||||
|
||||
Data Format
|
||||
-----------
|
||||
|
||||
This model expects the dataset in `SQuAD format`_ (i.e., a JSON file for each dataset split). The code snippet below shows an example
|
||||
of the training file. Each title has one or multiple paragraph entries, each consisting of the "context" and question-answer entries.
|
||||
Each question-answer entry has:
|
||||
|
||||
- A question
|
||||
- A globally unique id
|
||||
- The Boolean flag ``is_impossible``, which shows whether a question is answerable or not:
|
||||
- if the question is answerable, one answer entry contains the text span and its starting character index in the context
|
||||
- if the question is not answerable, an empty ``answers`` list is provided
|
||||
|
||||
.. _SQuAD format: https://rajpurkar.github.io/SQuAD-explorer/
|
||||
|
||||
The evaluation files (for validation and testing) follow the above format, except that it can provide more than one answer to the
|
||||
same question. The inference file also follows the above format, except that it does not require the ``answers`` and ``is_impossible``
|
||||
keywords.
|
||||
|
||||
The following is an example of the data format (JSON file):
|
||||
|
||||
.. code::
|
||||
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"title": "Super_Bowl_50",
|
||||
"paragraphs": [
|
||||
{
|
||||
"context": "Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24\u201310 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the \"golden anniversary\" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as \"Super Bowl L\"), so that the logo could prominently feature the Arabic numerals 50.",
|
||||
"qas": [
|
||||
{
|
||||
"question": "Where did Super Bowl 50 take place?",
|
||||
"is_impossible": "false",
|
||||
"id": "56be4db0acb8001400a502ee",
|
||||
"answers": [
|
||||
{
|
||||
"answer_start": "403",
|
||||
"text": "Santa Clara, California"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "What was the winning score of the Super Bowl 50?",
|
||||
"is_impossible": "true",
|
||||
"id": "56be4db0acb8001400a502ez",
|
||||
"answers": [
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
Dataset Download
|
||||
----------------
|
||||
|
||||
To perform training of the Question Answering model on the SQuAD dataset, you must first download it from `here
|
||||
<https://rajpurkar.github.io/SQuAD-explorer/>`_ or run:
|
||||
|
||||
.. code::
|
||||
|
||||
python get_squad.py
|
||||
|
||||
There are two versions:
|
||||
|
||||
- SQuAD version 1.1, which does not contain questions without the answer and has 100,000+ question-answer pairs on 500+ articles.
|
||||
- SQuAD version 2.0, which combines the 100,000 questions from SQuAD 1.1 with over 50,000 unanswerable questions. To do well with
|
||||
SQuAD 2.0, a system must not only answer questions when possible, but also determine when no answer is supported by the paragraph
|
||||
and abstain from answering.
|
||||
|
||||
After downloading the files, you should have a :code:`squad` data folder that contains the following four files for training and
|
||||
evaluation:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--squad
|
||||
|-- v1.1/train-v1.1.json
|
||||
|-- v1.1/dev-v1.1.json
|
||||
|-- v2.0/train-v2.0.json
|
||||
|-- v2.0/dev-v2.0.json
|
||||
|
||||
|
||||
.. _model_training_question_answering:
|
||||
|
||||
Model Training
|
||||
--------------
|
||||
|
||||
In the Question Answering Model, we are training a span prediction head on top of a pre-trained language model, such as
|
||||
`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ :cite:`nlp-qa-devlin2018bert`.
|
||||
Unless the user provides a pre-trained checkpoint for the language model, the language model is initialized with the pre-trained model
|
||||
from `HuggingFace Transformers <https://github.com/huggingface/transformers>`__.
|
||||
|
||||
Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/question_answering/conf/question_answering_squad_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/question_answering/conf/question_answering_squad_config.yaml>`__.
|
||||
|
||||
The specification can be grouped into three categories:
|
||||
|
||||
- Parameters that describe the training process: **trainer**
|
||||
- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds**, **model.test_ds**
|
||||
- Parameters that describe the model: **model**
|
||||
|
||||
More details about parameters in the spec file can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **pretrained_model** | string | Pretrained QA model model from ``list_available_models()`` or path to a ``.nemo`` file. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **do_training** | bool | If ``true``, starts training, otherwise, skips training and continues with evaluation/inference. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.version_2_with_negative** | bool | Set to ``true`` to allow examples without an answer, e.g. for SQuAD v2.0. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.do_lower_case** | bool | If ``true``, converts text to lower case, only import for inference/evaluation. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.use_cache** | bool | If ``true``, either loads all preprocessed data from cache or saves preprocessed data for future use. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.file** | string | The training file path. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.num_samples** | integer | The number of samples to use from the training dataset (use ``-1`` to specify all samples). |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.file** | string | The validation file path. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.num_samples** | integer | The number of samples to use from the validation dataset (use ``-1`` to specify all samples). |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **test_ds.file** | string | The test file path (optional). |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **test_ds.num_samples** | integer | The number of samples to use from the test dataset (use ``-1`` to specify all samples). |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
Example of the command for training the model:
|
||||
|
||||
.. code::
|
||||
|
||||
python question_answering_squad.py \
|
||||
model.train_ds.file=<PATH_TO_TRAIN_DATA_FILE> \
|
||||
model.validation_ds.file=<PATH_TO_VALIDATION_DATA_FILE> \
|
||||
model.dataset.version_2_with_negative=<ALLOW_UNANSWERABLE_SAMPLES> \
|
||||
model.dataset.do_lower_case=<DO_LOWER_CASE> \
|
||||
trainer.max_epochs=<NUM_EPOCHS> \
|
||||
trainer.gpus=[<CHANGE_TO_GPU(s)_YOU_WANT_TO_USE>]
|
||||
|
||||
.. Note::
|
||||
|
||||
The first time you train, it will take an extra 5-10 minutes to process the dataset. For future training runs, it will use the
|
||||
processed dataset if :code:`model.dataset.use_cache=true`, which is automatically cached in the files in the same directory as
|
||||
the data.
|
||||
|
||||
Required Arguments for Training
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`model.train_ds.file`: path to the training file in JSON format
|
||||
- :code:`model.validation_ds.file`: path to the validation file in JSON format
|
||||
|
||||
Fine-tuning Procedure
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Fine-tuning procedure and logs look similar to what's described in the Model Training section, with the addition of the model
|
||||
that is initially loaded from a previously trained checkpoint, e.g. by specifying :code:`pretrained_model=<PRETRAINED_MODEL_NAME>`.
|
||||
|
||||
Inference
|
||||
---------
|
||||
|
||||
An example script on how to run inference can be found at `examples/nlp/question_answering/question_answering_squad.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/question_answering/question_answering_squad.py>`_.
|
||||
|
||||
To run inference with the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python question_answering_squad.py \
|
||||
pretrained_model=<PRETRAINED_MODEL> \
|
||||
model.dataset.version_2_with_negative=<ALLOW_UNANSWERABLE_SAMPLES> \
|
||||
model.dataset.do_lower_case=<DO_LOWER_CASE> \
|
||||
do_training=false \
|
||||
model.validation_ds.file=<PATH_TO_INFERENCE_DATA_FILE>
|
||||
|
||||
Required Arguments for inference:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`pretrained_model`: pretrained QA Model model from ``list_available_models()`` or path to a ``.nemo`` file
|
||||
|
||||
Model Evaluation
|
||||
----------------
|
||||
|
||||
An example script on how to evaluate the pre-trained model, can be found at `examples/nlp/question_answering/question_answering_squad.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/question_answering/question_answering_squad.py>`_.
|
||||
|
||||
To run evaluation of the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python question_answering_squad.py \
|
||||
pretrained_model=<PRETRAINED_MODEL> \
|
||||
model.dataset.version_2_with_negative=<ALLOW_UNANSWERABLE_SAMPLES> \
|
||||
model.dataset.do_lower_case=<DO_LOWER_CASE> \
|
||||
do_training=false \
|
||||
model.test_ds.file=<PATH_TO_TEST_DATA_FILE>
|
||||
|
||||
|
||||
Required Arguments:
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`pretrained_model`: pretrained QA model from `list_available_models()`` or path to a ``.nemo`` file
|
||||
- :code:`model.test_ds.file`: path to test file
|
||||
|
||||
During evaluation of the :code:`test_ds`, the script generates the following metrics:
|
||||
|
||||
- :code:`Exact Match (EM)`
|
||||
- :code:`F1`
|
||||
|
||||
More details about these metrics can be found `here <https://en.wikipedia.org/wiki/F-score>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-QA
|
||||
:keyprefix: nlp-qa-
|
||||
@@ -0,0 +1,17 @@
|
||||
.. _sgd_qa:
|
||||
|
||||
Dialogue State Tracking - SGD-QA Model
|
||||
======================================
|
||||
|
||||
More details can be found in the paper
|
||||
`SGD-QA: Fast Schema-Guided Dialogue State Tracking for Unseen Services <https://arxiv.org/abs/2105.08049>`__ :cite:`nlp-sgdqa-zhang2021sgdqa`.
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/dialogue_state_tracking/sgd_qa.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/dialogue_state_tracking/sgd_qa.py>`__.
|
||||
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-SGDQA
|
||||
:keyprefix: nlp-sgdqa-
|
||||
@@ -0,0 +1,169 @@
|
||||
.. _text_classification:
|
||||
|
||||
Text Classification model
|
||||
=========================
|
||||
|
||||
Text Classification is a sequence classification model based on BERT-based encoders. It can be used for a
|
||||
variety of tasks like text classification, sentiment analysis, domain/intent detection for dialogue systems, etc.
|
||||
The model takes a text input and predicts a label/class for the whole sequence. Megatron-LM and most of the BERT-based encoders
|
||||
supported by HuggingFace including BERT, RoBERTa, and DistilBERT.
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/text_classification/text_classification_with_bert.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/text_classification/text_classification_with_bert.py>`__.
|
||||
The default configuration file for the model can be found at: `NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/text_classification/conf/text_classification_config.yaml>`__.
|
||||
|
||||
There is also a Jupyter notebook which explains how to work with this model. We recommend you try this model in the Jupyter notebook (can run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.):
|
||||
`NeMo/tutorials/nlp/Text_Classification_Sentiment_Analysis.ipynb <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Text_Classification_Sentiment_Analysis.ipynb>`__.
|
||||
This tutorial shows an example of how run the Text Classification model on a sentiment analysis task. You may connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator) to run the notebook.
|
||||
|
||||
Data Format
|
||||
-----------
|
||||
|
||||
The Text Classification model uses a simple text format as the dataset. It requires the data to be stored in TAB separated files
|
||||
(``.tsv``) with two columns: sentence and label. Each line of the data file contains text sequences, where words are separated with spaces and the label is separated with ``[TAB]``, i.e.:
|
||||
|
||||
.. code::
|
||||
|
||||
[WORD][SPACE][WORD][SPACE][WORD][TAB][LABEL]
|
||||
|
||||
Labels need to be integers starting from ``0``. Some examples taken from the SST2 dataset, which is a two-class dataset for sentiment analysis:
|
||||
|
||||
.. code::
|
||||
|
||||
saw how bad this movie was 0
|
||||
lend some dignity to a dumb story 0
|
||||
the greatest musicians 1
|
||||
|
||||
You may need separate files for train, validation, and test with this format.
|
||||
|
||||
Dataset Conversion
|
||||
------------------
|
||||
|
||||
If your dataset is stored in another format, you need to convert it to NeMo's format to use this model. There are some conversion
|
||||
scripts available for the following datasets:
|
||||
|
||||
- SST2 :cite:`nlp-textclassify-socher2013`
|
||||
- IMDB :cite:`nlp-textclassify-maas2011`
|
||||
- ChemProt :cite:`nlp-textclassify-lim2018chemical`
|
||||
- THUCnews :cite:`nlp-textclassify-li2007scalable`
|
||||
|
||||
You can convert them from their original format to NeMo's format. To convert the original datasets to NeMo's format, use the ``examples/text_classification/data/import_datasets.py`` script:
|
||||
|
||||
.. code:: python
|
||||
|
||||
python import_datasets.py \
|
||||
--dataset_name DATASET_NAME \
|
||||
--target_data_dir TARGET_PATH \
|
||||
--source_data_dir SOURCE_PATH
|
||||
|
||||
It reads the dataset specified by ``DATASET_NAME`` from ``SOURCE_PATH`` and converts it to NeMo's format. It then saves the new
|
||||
dataset at ``TARGET_PATH``.
|
||||
|
||||
Arguments:
|
||||
|
||||
- ``dataset_name``: name of the dataset to convert (``sst-2``, ``chemprot``, ``imdb``, and ``thucnews`` are currently supported)
|
||||
- ``source_data_dir``: directory of your dataset
|
||||
- ``target_data_dir``: directory to save the converted dataset
|
||||
|
||||
After the conversion, the ``TARGET_PATH`` should contain the following files:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--TARGET_PATH
|
||||
|-- train.tsv
|
||||
|-- dev.tsv
|
||||
|-- test.tsv
|
||||
|
||||
Some datasets do not have the test set or their test set does not have any labels, therefore, the corresponding file may be missing.
|
||||
|
||||
Model Training
|
||||
--------------
|
||||
|
||||
You may find an example of a config file to be used for training of the Text Classification model at `NeMo/examples/nlp/text_classification/conf/text_classification_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/text_classification/conf/text_classification_config.yaml>`__.
|
||||
You can change any of these parameters directly from the config file or update them with the command-line arguments.
|
||||
|
||||
The config file of the Text Classification model contains three main sections of ``trainer``, ``exp_manager``, and ``model``. You can
|
||||
find more details about the ``trainer`` and ``exp_manager`` at :doc:`./nlp_model`. Some sub-sections of the model section including
|
||||
``tokenizer``, ``language_model``, and ``optim`` are shared among most of the NLP models. The details of these sections can be found
|
||||
at :doc:`./nlp_model`.
|
||||
|
||||
Example of a command for training a Text Classification model on two GPUs for 50 epochs:
|
||||
|
||||
.. code::
|
||||
|
||||
python examples/nlp/text_classification/text_classification_with_bert.py \
|
||||
model.training_ds.file_path=<TRAIN_FILE_PATH> \
|
||||
model.validation_ds.file_path=<VALIDATION_FILE_PATH> \
|
||||
trainer.max_epochs=50 \
|
||||
trainer.gpus=[0,1] \
|
||||
optim.name=adam \
|
||||
optim.lr=0.0001 \
|
||||
model.nemo_path=<NEMO_FILE_PATH>
|
||||
|
||||
At the start of each training experiment, there is a printed log of the experiment specification including any parameters added or
|
||||
overridden via the command-line. It also shows additional information, such as which GPUs are available, where logs are saved,
|
||||
and some samples from the datasets with their corresponding inputs to the model. It also provides some stats on the lengths of
|
||||
sequences in the dataset.
|
||||
|
||||
After each epoch, you should see a summary table of metrics on the validation set which include the following metrics:
|
||||
|
||||
- :code:`Precision`
|
||||
- :code:`Recall`
|
||||
- :code:`F1`
|
||||
|
||||
At the end of training, NeMo saves the last checkpoint at the path specified by ``NEMO_FILE_PATH`` in ``.nemo`` format.
|
||||
|
||||
Model Arguments
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
The following table lists some of the model's parameters you can use in the config files or set them from the command-line when
|
||||
training a model:
|
||||
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Default** | **Description** |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.class_labels.class_labels_file** | string | ``null`` | Path to an optional file containing the labels; each line is the string label corresponding to a label. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.num_classes** | int | ``Required`` | Number of the categories or classes, ``0`` < ``Label`` < ``num_classes``. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.do_lower_case** | boolean | ``true`` for uncased models, ``false`` for cased | Specifies if inputs should be made lower case, would be set automatically if pre-trained model is used. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.max_seq_length** | int | ``256`` | Maximum length of the input sequences. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.class_balancing** | string | ``null`` | ``null`` or ``weighted_loss``. ``weighted_loss`` enables the weighted class balancing to handle unbalanced classes. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.use_cache** | boolean | ``false`` | Uses a cache to store the processed dataset, you can use it for large datasets for speed up. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.classifier_head.num_output_layers** | integer | ``2`` | Number of fully connected layers of the classifier on top of the BERT model. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **model.classifier_head.fc_dropout** | float | ``0.1`` | Dropout ratio of the fully connected layers. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.file_path** | string | ``Required`` | Path of the training ``.tsv`` file. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.batch_size** | integer | ``32`` | Data loader's batch size. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.num_workers** | integer | ``2`` | Number of worker threads for data loader. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.shuffle** | boolean | ``true`` (training), ``false`` (test and validation) | Shuffles data for each epoch. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.drop_last** | boolean | ``false`` | Specifies if last batch of data needs to get dropped if it is smaller than batch size. |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.pin_memory** | boolean | ``false`` | Enables pin_memory of PyTorch's data loader to enhance speed |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
| **{training,validation,test}_ds.num_samples** | integer | ``-1`` | Number of samples to be used from the dataset; -1 means all samples |
|
||||
+-----------------------------------------------+-----------------+--------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
|
||||
Model Evaluation and Inference
|
||||
------------------------------
|
||||
|
||||
After saving the model in ``.nemo`` format, you can load the model and perform evaluation or inference on the model. You can find
|
||||
some examples in the example script: `NeMo/examples/nlp/text_classification/text_classification_with_bert.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/text_classification/text_classification_with_bert.py>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-TEXTCLASSIFY
|
||||
:keyprefix: nlp-textclassify-
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
.. _token_classification:
|
||||
|
||||
Token Classification (Named Entity Recognition) Model
|
||||
=====================================================
|
||||
|
||||
Token Classification model supports named entity recognition (NER) and other token level classification tasks, as long as the data
|
||||
follows the format specified below.
|
||||
|
||||
We're going to use NER task throughout this section. NER, also referred to as entity chunking, identification or extraction, is the
|
||||
task of detecting and classifying key information (entities) in text. In other words, a NER model takes a piece of text as input and
|
||||
for each word in the text, the model identifies a category the word belongs to. For example, in a sentence: ``Mary lives in Santa Clara
|
||||
and works at NVIDIA``, the model should detect that ``Mary`` is a person, ``Santa Clara`` is a location and ``NVIDIA`` is a company.
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from nemo.collections.nlp.models import TokenClassificationModel
|
||||
|
||||
# to get the list of pre-trained models
|
||||
TokenClassificationModel.list_available_models()
|
||||
|
||||
# Download and load the pre-trained BERT-based model
|
||||
model = TokenClassificationModel.from_pretrained("ner_en_bert")
|
||||
|
||||
# try the model on a few examples
|
||||
model.add_predictions(['we bought four shirts from the nvidia gear store in santa clara.', 'NVIDIA is a company.'])
|
||||
|
||||
.. note::
|
||||
|
||||
We recommend you try this model in a Jupyter notebook (run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.):
|
||||
`NeMo/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb>`__.
|
||||
|
||||
Connect to an instance with a GPU (**Runtime** -> **Change runtime type** -> select **GPU** for the hardware accelerator).
|
||||
|
||||
An example script on how to train the model can be found here: `NeMo/examples/nlp/token_classification/token_classification_train.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/token_classification_train.py>`__.
|
||||
|
||||
An example script on how to run evaluation and inference can be found here: `NeMo/examples/nlp/token_classification/token_classification_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/token_classification_evaluate.py>`__.
|
||||
|
||||
The default configuration file for the model can be found here: `NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/conf/token_classification_config.yaml>`__.
|
||||
|
||||
.. _dataset_token_classification:
|
||||
|
||||
Data Input for Token Classification Model
|
||||
-----------------------------------------
|
||||
|
||||
For pre-training or fine-tuning of the model, the data should be split into 2 files:
|
||||
|
||||
- ``text.txt``
|
||||
- ``labels.txt``
|
||||
|
||||
Each line of the ``text.txt`` file contains text sequences, where words are separated with spaces, i.e.: ``[WORD] [SPACE] [WORD] [SPACE] [WORD]``.
|
||||
The ``labels.txt`` file contains corresponding labels for each word in ``text.txt``, the labels are separated with spaces, i.e.: ``[LABEL] [SPACE] [LABEL] [SPACE] [LABEL]``.
|
||||
Example of a ``text.txt`` file:
|
||||
|
||||
Jennifer is from New York City .
|
||||
She likes ...
|
||||
...
|
||||
|
||||
Corresponding ``labels.txt`` file:
|
||||
|
||||
B-PER O O B-LOC I-LOC I-LOC O
|
||||
O O ...
|
||||
...
|
||||
|
||||
Dataset Conversion
|
||||
------------------
|
||||
|
||||
To convert an `IOB format <https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging)>`__ (short for inside, outside, beginning) data to the format required for training, use
|
||||
`examples/nlp/token_classification/data/import_from_iob_format.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/data/import_from_iob_format.py)>`_.
|
||||
|
||||
.. code::
|
||||
|
||||
# For conversion from IOB format, for example, for CoNLL-2003 dataset:
|
||||
python import_from_iob_format.py --data_file=<PATH/TO/THE/FILE/IN/IOB/FORMAT>
|
||||
|
||||
Convert Dataset Required Arguments
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`--data_file`: path to the file to convert from IOB to NeMo format
|
||||
|
||||
After running the above command, the data directory, where the :code:`--data_file` is stored, should contain :code:`text_*.txt` and :code:`labels_*.txt` files.
|
||||
The default names for the training and evaluation in the :code:`conf/token_classification_config.yaml` are the following:
|
||||
|
||||
.. code::
|
||||
|
||||
.
|
||||
|--data_dir
|
||||
|-- labels_dev.txt
|
||||
|-- labels_train.txt
|
||||
|-- text_dev.txt
|
||||
|-- text_train.txt
|
||||
|
||||
|
||||
Training The Token Classification model
|
||||
---------------------------------------
|
||||
|
||||
In the Token Classification model, we are jointly training a classifier on top of a pre-trained language model, such as
|
||||
`BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding <https://arxiv.org/abs/1810.04805>`__ :cite:`nlp-ner-devlin2018bert`.
|
||||
Unless the user provides a pre-trained checkpoint for the language model, the language model is initialized with the pre-trained model
|
||||
from `HuggingFace Transformers <https://github.com/huggingface/transformers>`__.
|
||||
|
||||
Example of model configuration file for training the model can be found at: `NeMo/examples/nlp/token_classification/conf/token_classification_config.yaml <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/conf/token_classification_config.yaml>`__.
|
||||
|
||||
The specification can be roughly grouped into three categories:
|
||||
|
||||
- Parameters that describe the training process: **trainer**
|
||||
- Parameters that describe the datasets: **model.dataset**, **model.train_ds**, **model.validation_ds**
|
||||
- Parameters that describe the model: **model**
|
||||
|
||||
More details about parameters in the spec file can be found below:
|
||||
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **Parameter** | **Data Type** | **Description** |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.dataset.data_dir** | string | Path to the data converted to the specified above format. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.head.num_fc_layers** | integer | Number of fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.head.fc_dropout** | float | Dropout to apply to the input hidden states. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.head.activation** | string | Activation to use between fully connected layers. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **model.punct_head.use_transrormer_init** | bool | Whether to initialize the weights of the classifier head with the same approach used in Transformer. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.text_file** | string | Name of the text training file located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.labels_file** | string | Name of the labels training file located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **training_ds.num_samples** | integer | Number of samples to use from the training dataset, ``-1`` - to use all. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.text_file** | string | Name of the text file for evaluation, located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.labels_file** | string | Name of the labels dev file located at ``data_dir``. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
| **validation_ds.num_samples** | integer | Number of samples to use from the dev set, ``-1`` - to use all. |
|
||||
+-------------------------------------------+-----------------+--------------------------------------------------------------------------------------------------------------+
|
||||
|
||||
For more information, see :ref:`nlp_model`.
|
||||
|
||||
Example of the command for training the model:
|
||||
|
||||
.. code::
|
||||
|
||||
python token_classification_train.py \
|
||||
model.dataset.data_dir=<PATH_TO_DATA_DIR> \
|
||||
trainer.max_epochs=<NUM_EPOCHS> \
|
||||
trainer.gpus=[<CHANGE_TO_GPU(s)_YOU_WANT_TO_USE>]
|
||||
|
||||
|
||||
Required Arguments for Training
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`model.dataset.data_dir`: path to the directory with pre-processed data.
|
||||
|
||||
.. note::
|
||||
|
||||
While the arguments are defined in the spec file, if you want to override these parameter definitions in the spec file and
|
||||
experiment with them, use the command-line to define the parameter. For example, the sample spec file mentioned above has
|
||||
:code:`validation_ds.batch_size` set to ``64``. However, if the GPU utilization can be optimized further by
|
||||
using a larger batch size, override it to the desired value by adding the field :code:`validation_ds.batch_size=128` from
|
||||
the command-line. You can repeat this with any of the parameters defined in the sample spec file.
|
||||
|
||||
Inference
|
||||
---------
|
||||
|
||||
An example script on how to run inference can be found at `examples/nlp/token_classification/token_classification_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/token_classification_evaluate.py>`_.
|
||||
|
||||
To run inference with the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python token_classification_evaluate.py \
|
||||
pretrained_model=<PRETRAINED_MODEL>
|
||||
|
||||
Required Arguments for Inference
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`pretrained_model`: pretrained Token Classification model from ``list_available_models()`` or path to a ``.nemo`` file. For example, ``ner_en_bert`` or ``your_model.nemo``
|
||||
|
||||
Model Evaluation
|
||||
----------------
|
||||
|
||||
An example script on how to evaluate the pre-trained model can be found at `examples/nlp/token_classification/token_classification_evaluate.py <https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/token_classification/token_classification_evaluate.py>`_.
|
||||
|
||||
To start evaluation of the pre-trained model, run:
|
||||
|
||||
.. code::
|
||||
|
||||
python token_classification_evaluate.py \
|
||||
model.dataset.data_dir=<PATH/TO/DATA/DIR> \
|
||||
pretrained_model=ner_en_bert \
|
||||
model.test_ds.text_file=<text_*.txt> \
|
||||
model.test_ds.labels_file=<labels_*.txt> \
|
||||
model.dataset.max_seq_length=512
|
||||
|
||||
|
||||
Required Arguments
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
- :code:`pretrained_model`: pretrained Token Classification model from ``list_available_models()`` or path to a ``.nemo`` file. For example, ``ner_en_bert`` or ``your_model.nemo``
|
||||
- :code:`model.dataset.data_dir`: path to the directory that containes :code:`model.test_ds.text_file` and :code:`model.test_ds.labels_file`
|
||||
|
||||
During evaluation of the :code:`test_ds`, the script generates a classification report that includes the following metrics:
|
||||
|
||||
- :code:`Precision`
|
||||
- :code:`Recall`
|
||||
- :code:`F1`
|
||||
|
||||
For more information, see `here <https://en.wikipedia.org/wiki/Precision_and_recall>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: nlp_all.bib
|
||||
:style: plain
|
||||
:labelprefix: NLP-NER
|
||||
:keyprefix: nlp-ner-
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 107 KiB |
@@ -0,0 +1,312 @@
|
||||
.. _best-practices:
|
||||
|
||||
Best Practices
|
||||
==============
|
||||
|
||||
The NVIDIA NeMo Toolkit is available on GitHub as `open source <https://github.com/NVIDIA/NeMo>`_ as well as
|
||||
a `Docker container on NGC <https://ngc.nvidia.com/catalog/containers/nvidia:nemo>`_. It's assumed the user has
|
||||
already installed NeMo by following the :ref:`quick-start-guide` instructions.
|
||||
|
||||
The conversational AI pipeline consists of three major stages:
|
||||
|
||||
- Automatic Speech Recognition (ASR)
|
||||
- Natural Language Processing (NLP) or Natural Language Understanding (NLU)
|
||||
- Text-to-Speech (TTS) or voice synthesis
|
||||
|
||||
As you talk to a computer, the ASR phase converts the audio signal into text, the NLP stage interprets the question
|
||||
and generates a smart response, and finally the TTS phase converts the text into speech signals to generate audio for
|
||||
the user. The toolkit enables development and training of deep learning models involved in conversational AI and easily
|
||||
chain them together.
|
||||
|
||||
Why NeMo?
|
||||
---------
|
||||
|
||||
Deep learning model development for conversational AI is complex. It involves defining, building, and training several
|
||||
models in specific domains; experimenting several times to get high accuracy, fine tuning on multiple tasks and domain
|
||||
specific data, ensuring training performance and making sure the models are ready for deployment to inference applications.
|
||||
Neural modules are logical blocks of AI applications which take some typed inputs and produce certain typed outputs. By
|
||||
separating a model into its essential components in a building block manner, NeMo helps researchers develop state-of-the-art
|
||||
accuracy models for domain specific data faster and easier.
|
||||
|
||||
Collections of modules for core tasks as well as specific to speech recognition, natural language, speech synthesis help
|
||||
develop modular, flexible, and reusable pipelines.
|
||||
|
||||
A neural module’s inputs/outputs have a neural type, that describes the semantics, the axis order and meaning, and the dimensions
|
||||
of the input/output tensors. This typing allows neural modules to be safely chained together to build models for applications.
|
||||
|
||||
NeMo can be used to train new models or perform transfer learning on existing pre-trained models. Pre-trained weights per module
|
||||
(such as encoder, decoder) help accelerate model training for domain specific data.
|
||||
|
||||
ASR, NLP and TTS pre-trained models are trained on multiple datasets (including some languages such as Mandarin) and optimized
|
||||
for high accuracy. They can be used for transfer learning as well.
|
||||
|
||||
NeMo supports developing models that work with Mandarin Chinese data. Tutorials help users train or fine tune models for
|
||||
conversational AI with the Mandarin Chinese language. The export method provided in NeMo makes it easy to transform a trained
|
||||
model into inference ready format for deployment.
|
||||
|
||||
A key area of development in the toolkit is interoperability with other tools used by speech researchers. Data layer for Kaldi
|
||||
compatibility is one such example.
|
||||
|
||||
NeMo, PyTorch Lightning, And Hydra
|
||||
----------------------------------
|
||||
|
||||
Conversational AI architectures are typically very large and require a lot of data and compute for training. NeMo uses
|
||||
`Pytorch Lightning <https://github.com/PyTorchLightning/pytorch-lightning>`_ for easy and performant multi-GPU/multi-node
|
||||
mixed precision training.
|
||||
|
||||
Pytorch Lightning is a high-performance PyTorch wrapper that organizes PyTorch code, scales model training, and reduces
|
||||
boilerplate. PyTorch Lightning has two main components, the ``LightningModule`` and the Trainer. The ``LightningModule`` is
|
||||
used to organize PyTorch code so that deep learning experiments can be easily understood and reproduced. The Pytorch Lightning
|
||||
Trainer is then able to take the ``LightningModule`` and automate everything needed for deep learning training.
|
||||
|
||||
NeMo models are LightningModules that come equipped with all supporting infrastructure for training and reproducibility. This
|
||||
includes the deep learning model architecture, data preprocessing, optimizer, check-pointing and experiment logging. NeMo
|
||||
models, like LightningModules, are also PyTorch modules and are fully compatible with the broader PyTorch ecosystem. Any NeMo
|
||||
model can be taken and plugged into any PyTorch workflow.
|
||||
|
||||
Configuring conversational AI applications is difficult due to the need to bring together many different Python libraries into
|
||||
one end-to-end system. NeMo uses Hydra for configuring both NeMo models and the PyTorch Lightning Trainer. `Hydra <https://github.com/facebookresearch/hydra>`_
|
||||
is a flexible solution that makes it easy to configure all of these libraries from a configuration file or from the command-line.
|
||||
|
||||
Every NeMo model has an example configuration file and a corresponding script that contains all configurations needed for training
|
||||
to state-of-the-art accuracy. NeMo models have the same look and feel so that it is easy to do conversational AI research across
|
||||
multiple domains.
|
||||
|
||||
Using Optimized Pretrained Models With NeMo
|
||||
-------------------------------------------
|
||||
|
||||
`NVIDIA GPU Cloud (NGC) <https://ngc.nvidia.com/catalog>`_ is a software repository that has containers and models optimized
|
||||
for deep learning. NGC hosts many conversational AI models developed with NeMo that have been trained to state-of-the-art accuracy
|
||||
on large datasets. NeMo models on NGC can be automatically downloaded and used for transfer learning tasks. Pretrained models
|
||||
are the quickest way to get started with conversational AI on your own data. NeMo has many `example scripts <https://github.com/NVIDIA/NeMo/tree/main/examples>`_
|
||||
and `Jupyter Notebook tutorials <https://github.com/NVIDIA/NeMo#tutorials>`_ showing step-by-step how to fine-tune pretrained NeMo
|
||||
models on your own domain-specific datasets.
|
||||
|
||||
For BERT based models, the model weights provided are ready for
|
||||
downstream NLU tasks. For speech models, it can be helpful to start with a pretrained model and then continue pretraining on your
|
||||
own domain-specific data. Jasper and QuartzNet base model pretrained weights have been known to be very efficient when used as
|
||||
base models. For an easy to follow guide on transfer learning and building domain specific ASR models, you can follow this `blog <https://developer.nvidia.com/blog/how-to-build-domain-specific-automatic-speech-recognition-models-on-gpus/>`_.
|
||||
All pre-trained NeMo models can be found on the `NGC NeMo Collection <https://ngc.nvidia.com/catalog/collections?orderBy=scoreDESC&pageNumber=0&query=NeMo&quickFilter=&filters=>`_. Everything needed to quickly get started
|
||||
with NeMo ASR, NLP, and TTS models is there.
|
||||
|
||||
Pre-trained models are packaged as a ``.nemo`` file and contain the PyTorch checkpoint along with everything needed to use the model.
|
||||
NeMo models are trained to state-of-the-art accuracy and trained on multiple datasets so that they are robust to small differences
|
||||
in data. NeMo contains a large variety of models such as speaker identification and Megatron BERT and the best models in speech and
|
||||
language are constantly being added as they become available. NeMo is the premier toolkit for conversational AI model building and
|
||||
training.
|
||||
|
||||
For a list of supported models, refer to the :ref:`tutorials` section.
|
||||
|
||||
ASR Guidance
|
||||
------------
|
||||
|
||||
This section is to help guide your decision making by answering our most asked ASR questions.
|
||||
|
||||
**Q: Is there a way to add domain specific vocabulary in NeMo? If so, how do I do that?**
|
||||
A: QuartzNet and Jasper models are character-based. So pretrained models we provide for these two output lowercase English
|
||||
letters and ‘. Users can re-retrain them on vocabulary with upper case letters and punctuation symbols.
|
||||
|
||||
**Q: When training, there are “Reference” lines and “Decoded” lines that are printed out. It seems like the reference line should
|
||||
be the “truth” line and the decoded line should be what the ASR is transcribing. Why do I see that even the reference lines do not
|
||||
appear to be correct?**
|
||||
A: Because our pre-trained models can only output lowercase letters and apostrophe, everything else is dropped. So the model will
|
||||
transcribe 10 as ten. The best way forward is to prepare the training data first by transforming everything to lowercase and convert
|
||||
the numbers from digit representation to word representation using a simple library such as `inflect <https://pypi.org/project/inflect/>`_. Then, add the uppercase letters
|
||||
and punctuation back using the NLP punctuation model. Here is an example of how this is incorporated: `NeMo voice swap demo <https://github.com/NVIDIA/NeMo/blob/main/tutorials/VoiceSwapSample.ipynb>`_.
|
||||
|
||||
**Q: What languages are supported in NeMo currently?**
|
||||
A: Along with English, we provide pre-trained models for Zh, Es, Fr, De, Ru, It, Ca and Pl languages.
|
||||
For more information, see `NeMo Speech Models <https://ngc.nvidia.com/catalog/collections/nvidia:nemo_asr>`_.
|
||||
|
||||
Data Augmentation
|
||||
-----------------
|
||||
|
||||
Data augmentation in ASR is invaluable. It comes at the cost of increased training time if samples are augmented during training
|
||||
time. To save training time, it is recommended to pre-process the dataset offline for a one time preprocessing cost and then train
|
||||
the dataset on this augmented training set.
|
||||
|
||||
For example, processing a single sample involves:
|
||||
|
||||
- Speed perturbation
|
||||
- Time stretch perturbation (sample level)
|
||||
- Noise perturbation
|
||||
- Impulse perturbation
|
||||
- Time stretch augmentation (batch level, neural module)
|
||||
|
||||
A simple tutorial guides users on how to use these utilities provided in `GitHub: NeMo <https://github.com/NVIDIA/NeMo/blob/main/tutorials/asr/05_Online_Noise_Augmentation.ipynb>`_.
|
||||
|
||||
Speech Data Explorer
|
||||
--------------------
|
||||
|
||||
Speech data explorer is a `Dash-based tool <https://plotly.com/dash/>`_ for interactive exploration of ASR/TTS datasets.
|
||||
|
||||
Speech data explorer collects:
|
||||
|
||||
- dataset statistics (alphabet, vocabulary, and duration-based histograms)
|
||||
- navigation across datasets (sorting and filtering)
|
||||
- inspections of individual utterances (waveform, spectrogram, and audio player)
|
||||
- errors analysis (word error rate, character error rate, word match rate, mean word accuracy, and diff)
|
||||
|
||||
In order to use the tool, it needs to be installed separately. Perform the steps `here <https://github.com/NVIDIA/NeMo/tree/main/tools/speech_data_explorer>`_ to install speech data explorer.
|
||||
|
||||
Using Kaldi Formatted Data
|
||||
--------------------------
|
||||
|
||||
The `Kaldi Speech Recognition Toolkit <https://kaldi-asr.org/>`_ project began in 2009 at `Johns Hopkins University <https://www.jhu.edu/>`. It is a toolkit written in C++. If
|
||||
researchers have used Kaldi and have datasets that are formatted to be used with the toolkit; they can use NeMo to develop models
|
||||
based on that data.
|
||||
|
||||
To load Kaldi-formatted data, you can simply use ``KaldiFeatureDataLayer`` instead of ``AudioToTextDataLayer``. The ``KaldiFeatureDataLayer``
|
||||
takes in the argument ``kaldi_dir`` instead of a ``manifest_filepath``. The ``manifest_filepath`` argument should be set to the directory
|
||||
that contains the files ``feats.scp`` and ``text``.
|
||||
|
||||
Using Speech Command Recognition Task For ASR Models
|
||||
----------------------------------------------------
|
||||
|
||||
Speech Command Recognition is the task of classifying an input audio pattern into a set of discrete classes. It is a subset of ASR,
|
||||
sometimes referred to as Key Word Spotting, in which a model is constantly analyzing speech patterns to detect certain ``action`` classes.
|
||||
|
||||
Upon detection of these commands, a specific action can be taken. An example Jupyter notebook provided in NeMo shows how to train a
|
||||
QuartzNet model with a modified decoder head trained on a speech commands dataset.
|
||||
|
||||
.. note:: It is preferred that you use absolute paths to ``data_dir`` when preprocessing the dataset.
|
||||
|
||||
NLP Fine-Tuning BERT
|
||||
--------------------
|
||||
|
||||
BERT, or Bidirectional Encoder Representations from Transformers, is a neural approach to pre-train language representations which
|
||||
obtains near state-of-the-art results on a wide array of Natural Language Processing (NLP) tasks, including the GLUE benchmark and
|
||||
SQuAD Question & Answering dataset.
|
||||
|
||||
BERT model checkpoints (`BERT-large-uncased <https://ngc.nvidia.com/catalog/models/nvidia:bertlargeuncasedfornemo>`_ and `BERT-base-uncased <https://ngc.nvidia.com/catalog/models/nvidia:bertbaseuncasedfornemo>`_) are provided can be used for either fine tuning BERT on your custom
|
||||
dataset, or fine tuning downstream tasks, including GLUE benchmark tasks, Question & Answering tasks, Joint Intent & Slot detection,
|
||||
Punctuation and Capitalization, Named Entity Recognition, and Speech Recognition post processing model to correct mistakes.
|
||||
|
||||
.. note:: Almost all NLP examples also support RoBERTa and ALBERT models for downstream fine-tuning tasks (see the list of all
|
||||
supported models by calling ``nemo.collections.nlp.modules.common.lm_utils.get_pretrained_lm_models_list()``. The user needs to specify
|
||||
the name of the model desired while running the example scripts.
|
||||
|
||||
BioMegatron Medical BERT
|
||||
------------------------
|
||||
|
||||
BioMegatron is a large language model (Megatron-LM) trained on larger domain text corpus (PubMed abstract + full-text-commercial).
|
||||
It achieves state-of-the-art results for certain tasks such as Relationship Extraction, Named Entity Recognition and Question &
|
||||
Answering. Follow these tutorials to learn how to train and fine tune BioMegatron; pretrained models are provided on NGC:
|
||||
|
||||
- `Relation Extraction BioMegatron <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb>`_
|
||||
- `Token Classification BioMegatron <https://github.com/NVIDIA/NeMo/blob/main/tutorials/nlp/Token_Classification-BioMegatron.ipynb>`_
|
||||
|
||||
Efficient Training With NeMo
|
||||
----------------------------
|
||||
|
||||
Using Mixed Precision
|
||||
^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Mixed precision accelerates training speed while protecting against noticeable loss. Tensor Cores is a specific hardware unit that
|
||||
comes starting with the Volta and Turing architectures to accelerate large matrix to matrix multiply-add operations by operating them
|
||||
on half precision inputs and returning the result in full precision.
|
||||
|
||||
Neural networks which usually use massive matrix multiplications can be significantly sped up with mixed precision and Tensor Cores.
|
||||
However, some neural network layers are numerically more sensitive than others. Apex AMP is an NVIDIA library that maximizes the
|
||||
benefit of mixed precision and Tensor Cores usage for a given network.
|
||||
|
||||
Multi-GPU Training
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
This section is to help guide your decision making by answering our most asked multi-GPU training questions.
|
||||
|
||||
**Q: Why is multi-GPU training preferred over other types of training?**
|
||||
A: Multi-GPU training can reduce the total training time by distributing the workload onto multiple compute instances. This is
|
||||
particularly important for large neural networks which would otherwise take weeks to train until convergence. Since NeMo supports
|
||||
multi-GPU training, no code change is needed to move from single to multi-GPU training, only a slight change in your launch command
|
||||
is required.
|
||||
|
||||
**Q: What are the advantages of mixed precision training?**
|
||||
A: Mixed precision accelerates training speed while protecting against noticeable loss in precision. Tensor Cores is a specific
|
||||
hardware unit that comes starting with the Volta and Turing architectures to accelerate large matrix multiply-add operations by
|
||||
operating on half precision inputs and returning the result in full precision in order to prevent loss in precision. Neural
|
||||
networks which usually use massive matrix multiplications can be significantly sped up with mixed precision and Tensor Cores.
|
||||
However, some neural network layers are numerically more sensitive than others. Apex AMP is a NVIDIA library that maximizes the
|
||||
benefit of mixed precision and Tensor Core usage for a given network.
|
||||
|
||||
**Q: What is the difference between multi-GPU and multi-node training?**
|
||||
A: Multi-node is an abstraction of multi-GPU training, which requires a distributed compute cluster, where each node can have multiple
|
||||
GPUs. Multi-node training is needed to scale training beyond a single node to large amounts of GPUs.
|
||||
|
||||
From the framework perspective, nothing changes from moving to multi-node training. However, a master address and port needs to be set
|
||||
up for inter-node communication. Multi-GPU training will then be launched on each node with passed information. You might also consider
|
||||
the underlying inter-node network topology and type to achieve full performance, such as HPC-style hardware such as NVLink, InfiniBand
|
||||
networking, or Ethernet.
|
||||
|
||||
|
||||
Recommendations For Optimization And FAQs
|
||||
-----------------------------------------
|
||||
|
||||
This section is to help guide your decision making by answering our most asked NeMo questions.
|
||||
|
||||
**Q: Are there areas where performance can be increased?**
|
||||
A: You should try using mixed precision for improved performance. Note that typically when using mixed precision, memory consumption
|
||||
is decreased and larger batch sizes could be used to further improve the performance.
|
||||
|
||||
When fine-tuning ASR models on your data, it is almost always possible to take advantage of NeMo's pre-trained modules. Even if you
|
||||
have a different target vocabulary, or even a different language; you can still try starting with pre-trained weights from Jasper or
|
||||
QuartzNet ``encoder`` and only adjust the ``decoder`` for your needs.
|
||||
|
||||
**Q: What is the recommended sampling rate for ASR?**
|
||||
A: The released models are based on 16 KHz audio, therefore, ensure you use models with 16 KHz audio. Reduced performance should be
|
||||
expected for any audio that is up-sampled from a sampling frequency less than 16 KHz data.
|
||||
|
||||
**Q: How do we use this toolkit for audio with different types of compression and frequency than the training domain for ASR?**
|
||||
A: You have to match the compression and frequency.
|
||||
|
||||
**Q: How do you replace the 6-gram out of the ASR model with a custom language model? What is the language format supported in NeMo?**
|
||||
A: NeMo’s Beam Search decoder with Levenberg-Marquardt (LM) neural module supports the KenLM language model.
|
||||
|
||||
- You should retrain the KenLM language model on your own dataset. Refer to `KenLM’s documentation <https://github.com/kpu/kenlm#kenlm>`_.
|
||||
- If you want to use a different language model, other than KenLM, you will need to implement a corresponding decoder module.
|
||||
- Transformer-XL example is present in OS2S. It would need to be updated to work with NeMo. `Here is the code <https://github.com/NVIDIA/OpenSeq2Seq/tree/master/external_lm_rescore>`_.
|
||||
|
||||
**Q: How do I use text-to-speech (TTS)?**
|
||||
A:
|
||||
- Obtain speech data ideally at 22050 Hz or alternatively at a higher sample rate and then down sample to 22050 Hz.
|
||||
- If less than 22050 Hz and above 16000 Hz:
|
||||
- Retrain WaveGlow on your own dataset.
|
||||
- Tweak the spectrogram generation parameters, namely the ``window_size`` and the ``window_stride`` for their fourier
|
||||
transforms.
|
||||
- For below 16000 Hz, look into obtaining new data.
|
||||
- In terms of bitrate/quantization, the general advice is the higher the better. We have not experimented enough to state how much
|
||||
this impacts quality.
|
||||
- For the amount of data, again the more the better, and the more diverse in terms of phonemes the better. Aim for around 20 hours
|
||||
of speech after filtering for silences and non-speech audio.
|
||||
- Most open speech datasets are in ~10 second format so training Tacotron 2 on audio on the order of 10s - 20s per sample is known
|
||||
to work. Additionally, the longer the speech samples, the more difficult it will be to train Tacotron 2.
|
||||
- Audio files should be clean. There should be little background noise or music. Data recorded from a studio mic is likely to be easier
|
||||
to train compared to data captured using a phone.
|
||||
- To ensure pronunciation of words are accurate; the technical challenge is related to the dataset, text to phonetic spelling is
|
||||
required, use phonetic alphabet (notation) that has the name correctly pronounced.
|
||||
- Here are some example parameters you can use to train Tacotron 2:
|
||||
- Tacotron 2 requires a single speaker dataset
|
||||
- Use AMP level O0
|
||||
- Trim long silences in the beginning and end
|
||||
- ``optimizer="adam"``
|
||||
- ``beta1 = 0.9``
|
||||
- ``beta2 = 0.999``
|
||||
- ``lr=0.001 (constant)``
|
||||
- ``amp_opt_level="O0"``
|
||||
- ``weight_decay=1e-6``
|
||||
- ``batch_size=48 (per GPU)``
|
||||
- ``trim_silence=True``
|
||||
|
||||
Resources
|
||||
---------
|
||||
|
||||
Ensure you are familiar with the following resources for NeMo.
|
||||
|
||||
- Developer blogs
|
||||
- `How to Build Domain Specific Automatic Speech Recognition Models on GPUs <https://developer.nvidia.com/blog/how-to-build-domain-specific-automatic-speech-recognition-models-on-gpus/>`_
|
||||
- `Develop Smaller Speech Recognition Models with NVIDIA’s NeMo Framework <https://developer.nvidia.com/blog/develop-smaller-speech-recognition-models-with-nvidias-nemo-framework/>`_
|
||||
- `Neural Modules for Fast Development of Speech and Language Models <https://developer.nvidia.com/blog/neural-modules-for-speech-language-models/>`_
|
||||
|
||||
- Domain specific, transfer learning, Docker container with Jupyter Notebooks
|
||||
- `Domain Specific NeMo ASR Application <https://ngc.nvidia.com/catalog/containers/nvidia:nemo_asr_app_img>`_
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
Introduction
|
||||
============
|
||||
|
||||
.. # define a hard line break for html
|
||||
.. |br| raw:: html
|
||||
|
||||
<br />
|
||||
|
||||
.. _dummy_header:
|
||||
|
||||
`NVIDIA NeMo <https://github.com/NVIDIA/NeMo>`_ is a toolkit for building new state-of-the-art
|
||||
conversational AI models. NeMo has separate collections for Automatic Speech Recognition (ASR),
|
||||
Natural Language Processing (NLP), and Text-to-Speech (TTS) models. Each collection consists of
|
||||
prebuilt modules that include everything needed to train on your data.
|
||||
Every module can easily be customized, extended, and composed to create new conversational AI
|
||||
model architectures.
|
||||
|
||||
Conversational AI architectures are typically large and require a lot of data and compute
|
||||
for training. NeMo uses `PyTorch Lightning <https://www.pytorchlightning.ai/>`_ for easy and performant multi-GPU/multi-node
|
||||
mixed-precision training.
|
||||
|
||||
.. raw:: html
|
||||
|
||||
<div style="position: relative; padding-bottom: 3%; height: 0; overflow: hidden; max-width: 100%; height: auto;">
|
||||
<iframe width="560" height="315" src="https://www.youtube.com/embed/wBgpMf_KQVw" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
|
||||
</div>
|
||||
|
||||
For more information and questions, visit the `NVIDIA NeMo Discussion Board <https://github.com/NVIDIA/NeMo/discussions>`_.
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
|
||||
Before you begin using NeMo, it's assumed you meet the following prerequisites.
|
||||
|
||||
#. You have Python version 3.6, 3.7 or 3.8.
|
||||
|
||||
#. You have Pytorch version 1.8.1.
|
||||
|
||||
#. You have access to an NVIDIA GPU for training.
|
||||
|
||||
.. _quick_start_guide:
|
||||
|
||||
Quick Start Guide
|
||||
-----------------
|
||||
|
||||
This NeMo Quick Start Guide is a starting point for users who want to try out NeMo; specifically, this guide enables users to quickly get started with the NeMo fundamentals by walking you through an example audio translator and voice swap.
|
||||
|
||||
If you're new to NeMo, the best way to get started is to take a look at the following tutorials:
|
||||
|
||||
* `Text Classification (Sentiment Analysis) <https://github.com/NVIDIA/NeMo/blob/main/tutorials/Text_Classification_Sentiment_Analysis>`__ - demonstrates the Text Classification model using the NeMo NLP collection.
|
||||
* `NeMo Primer <https://github.com/NVIDIA/NeMo/blob/main/tutorials/00_NeMo_Primer.ipynb>`__ - introduces NeMo, PyTorch Lightning, and OmegaConf, and shows how to use, modify, save, and restore NeMo models.
|
||||
* `NeMo Models <https://github.com/NVIDIA/NeMo/blob/main/tutorials/01_NeMo_Models.ipynb>`__ - explains the fundamental concepts of the NeMo model.
|
||||
* `NeMo voice swap demo <https://github.com/NVIDIA/NeMo/blob/main/tutorials/NeMo_voice_swap_app.ipynb>`__ - demonstrates how to swap a voice in the audio fragment with a computer generated one using NeMo.
|
||||
|
||||
Below we is the code snippet of Audio Translator application.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Import NeMo and it's ASR, NLP and TTS collections
|
||||
import nemo
|
||||
# Import Speech Recognition collection
|
||||
import nemo.collections.asr as nemo_asr
|
||||
# Import Natural Language Processing colleciton
|
||||
import nemo.collections.nlp as nemo_nlp
|
||||
# Import Speech Synthesis collection
|
||||
import nemo.collections.tts as nemo_tts
|
||||
|
||||
# Next, we instantiate all the necessary models directly from NVIDIA NGC
|
||||
# Speech Recognition model - QuartzNet trained on Russian part of MCV 6.0
|
||||
quartznet = nemo_asr.models.EncDecCTCModel.from_pretrained(model_name="stt_ru_quartznet15x5").cuda()
|
||||
# Neural Machine Translation model
|
||||
nmt_model = nemo_nlp.models.MTEncDecModel.from_pretrained(model_name='nmt_ru_en_transformer6x6').cuda()
|
||||
# Spectrogram generator which takes text as an input and produces spectrogram
|
||||
spectrogram_generator = nemo_tts.models.Tacotron2Model.from_pretrained(model_name="tts_en_tacotron2").cuda()
|
||||
# Vocoder model which takes spectrogram and produces actual audio
|
||||
vocoder = nemo_tts.models.WaveGlowModel.from_pretrained(model_name="tts_waveglow_88m").cuda()
|
||||
# Transcribe an audio file
|
||||
# IMPORTANT: The audio must be mono with 16Khz sampling rate
|
||||
# Get example from: https://nemo-public.s3.us-east-2.amazonaws.com/mcv-samples-ru/common_voice_ru_19034087.wav
|
||||
russian_text = quartznet.transcribe(['Path_to_audio_file'])
|
||||
print(russian_text)
|
||||
# You should see russian text here. Let's translate it to English
|
||||
english_text = nmt_model.translate(russian_text)
|
||||
print(english_text)
|
||||
# After this you should see English translation
|
||||
# Let's convert it into audio
|
||||
# A helper function which combines Tacotron2 and WaveGlow to go directly from
|
||||
# text to audio
|
||||
def text_to_audio(text):
|
||||
parsed = spectrogram_generator.parse(text)
|
||||
spectrogram = spectrogram_generator.generate_spectrogram(tokens=parsed)
|
||||
audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram)
|
||||
return audio.to('cpu').numpy()
|
||||
audio = text_to_audio(english_text[0])
|
||||
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
Pip
|
||||
~~~
|
||||
Use this installation mode if you want the latest released version.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
apt-get update && apt-get install -y libsndfile1 ffmpeg
|
||||
pip install Cython
|
||||
pip install nemo_toolkit[all]
|
||||
|
||||
Pip from source
|
||||
~~~~~~~~~~~~~~~
|
||||
Use this installation mode if you want the version from a particular GitHub branch (for example, ``main``).
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
apt-get update && apt-get install -y libsndfile1 ffmpeg
|
||||
pip install Cython
|
||||
python -m pip install git+https://github.com/NVIDIA/NeMo.git@{BRANCH}#egg=nemo_toolkit[all]
|
||||
# For v1.0.2, replace {BRANCH} with v1.0.2 like so:
|
||||
# python -m pip install git+https://github.com/NVIDIA/NeMo.git@v1.0.2#egg=nemo_toolkit[all]
|
||||
|
||||
From source
|
||||
~~~~~~~~~~~
|
||||
Use this installation mode if you are contributing to NeMo.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
apt-get update && apt-get install -y libsndfile1 ffmpeg
|
||||
git clone https://github.com/NVIDIA/NeMo
|
||||
cd NeMo
|
||||
./reinstall.sh
|
||||
|
||||
Docker containers
|
||||
~~~~~~~~~~~~~~~~~
|
||||
If you chose to work with the ``main`` branch, we recommend using `NVIDIA's PyTorch container version 21.05-py3 <https://ngc.nvidia.com/containers/nvidia:pytorch/tags>`_, then install from GitHub.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker run --gpus all -it --rm -v <nemo_github_folder>:/NeMo --shm-size=8g \
|
||||
-p 8888:8888 -p 6006:6006 --ulimit memlock=-1 --ulimit \
|
||||
stack=67108864 --device=/dev/snd nvcr.io/nvidia/pytorch:21.05-py3
|
||||
|
||||
|
||||
FAQ
|
||||
---
|
||||
Have a look at our `discussions board <https://github.com/NVIDIA/NeMo/discussions>`_ and feel free to post a question or start a discussion.
|
||||
|
||||
|
||||
Contributing
|
||||
------------
|
||||
|
||||
We welcome community contributions! Refer to the `CONTRIBUTING.md <https://github.com/NVIDIA/NeMo/blob/main/CONTRIBUTING.md>`_ file for the process.
|
||||
|
||||
License
|
||||
-------
|
||||
|
||||
NeMo is under `Apache 2.0 license <https://github.com/NVIDIA/NeMo/blob/main/LICENSE>`_.
|
||||
@@ -0,0 +1,117 @@
|
||||
.. _tutorials:
|
||||
|
||||
Tutorials
|
||||
=========
|
||||
|
||||
The best way to get started with NeMo is to start wtih one of our tutorials.
|
||||
|
||||
Most NeMo tutorials can be run on `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_.
|
||||
|
||||
To run a tutorial:
|
||||
|
||||
#. Click the **Colab** link (see table below).
|
||||
#. Connect to an instance with a GPU. For example, click **Runtime** > **Change runtime type** and select **GPU** for the hardware accelerator.
|
||||
|
||||
.. list-table:: **Tutorials**
|
||||
:widths: 15 25 25
|
||||
:header-rows: 1
|
||||
|
||||
* - Domain
|
||||
- Title
|
||||
- GitHub URL
|
||||
* - General
|
||||
- Getting Started: Exploring Nemo Fundamentals
|
||||
- `NeMo Fundamentals <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/00_NeMo_Primer.ipynb>`_
|
||||
* - General
|
||||
- Getting Started: Sample Conversational AI application
|
||||
- `Audio translator example <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/AudioTranslationSample.ipynb>`_
|
||||
* - General
|
||||
- Getting Started: Voice swap application
|
||||
- `Voice swap example <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/VoiceSwapSample.ipynb>`_
|
||||
* - General
|
||||
- Exploring NeMo Model Construction
|
||||
- `NeMo Models <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/01_NeMo_Models.ipynb>`_
|
||||
* - ASR
|
||||
- ASR with NeMo
|
||||
- `ASR with NeMo <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/01_ASR_with_NeMo.ipynb>`_
|
||||
* - ASR
|
||||
- ASR with Subword Tokenization
|
||||
- `ASR with Subword Tokenization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/08_ASR_with_Subword_Tokenization.ipynb>`_
|
||||
* - ASR
|
||||
- Offline ASR Inference with Beam Search and External Language Model Rescoring
|
||||
- `Offline ASR <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/Offline_ASR.ipynb>`_
|
||||
* - ASR
|
||||
- Online ASR inference with Microphone
|
||||
- `Online ASR Microphone <https://github.com/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/02_Online_ASR_Microphone_Demo.ipynb>`_
|
||||
* - ASR
|
||||
- Fine-tuning CTC Models on New Languages
|
||||
- `ASR CTC Language Fine-Tuning <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/asr/10_ASR_CTC_Language_Finetuning.ipynb>`_
|
||||
* - ASR
|
||||
- Speech Commands
|
||||
- `Speech Commands <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/03_Speech_Commands.ipynb>`_
|
||||
* - ASR
|
||||
- Online and Offline Speech Commands Inference
|
||||
- `Online Offline Microphone Speech Commands <https://github.com/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/04_Online_Offline_Speech_Commands_Demo.ipynb>`_
|
||||
* - ASR
|
||||
- Voice Activiy Detection (VAD)
|
||||
- `Voice Activiy Detection <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/06_Voice_Activiy_Detection.ipynb>`_
|
||||
* - ASR
|
||||
- Online and Offline VAD Inference
|
||||
- `Online Offline Microphone VAD <https://github.com/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/07_Online_Offline_Microphone_VAD_Demo.ipynb>`_
|
||||
* - ASR
|
||||
- Speaker Recognition and Verification
|
||||
- `Speaker Recognition and Verification <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/speaker_recognition/Speaker_Recognition_Verification.ipynb>`_
|
||||
* - ASR
|
||||
- Speaker Diarization Inference
|
||||
- `Speaker Diarization Inference <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/speaker_recognition/Speaker_Diarization_Inference.ipynb>`_
|
||||
* - ASR
|
||||
- ASR with Speaker Diarization
|
||||
- `ASR with Speaker Diarization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/speaker_recognition/ASR_with_SpeakerDiarization.ipynb>`_
|
||||
* - ASR
|
||||
- Online Noise Augmentation
|
||||
- `Online Noise Augmentation <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/asr/05_Online_Noise_Augmentation.ipynb>`_
|
||||
* - NLP
|
||||
- Using Pretrained Language Models for Downstream Tasks
|
||||
- `Pretrained Language Models for Downstream Tasks <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/01_Pretrained_Language_Models_for_Downstream_Tasks.ipynb>`_
|
||||
* - NLP
|
||||
- Exploring NeMo NLP Tokenizers
|
||||
- `NLP Tokenizers <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/02_NLP_Tokenizers.ipynb>`_
|
||||
* - NLP
|
||||
- Text Classification (Sentiment Analysis) with BERT
|
||||
- `Text Classification (Sentiment Analysis) <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Text_Classification_Sentiment_Analysis.ipynb>`_
|
||||
* - NLP
|
||||
- Question Answering with SQuAD
|
||||
- `Question Answering Squad <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Question_Answering_Squad.ipynb>`_
|
||||
* - NLP
|
||||
- Token Classification (Named Entity Recognition)
|
||||
- `Token Classification: Named Entity Recognition <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Token_Classification_Named_Entity_Recognition.ipynb>`_
|
||||
* - NLP
|
||||
- Joint Intent Classification and Slot Filling
|
||||
- `Joint Intent and Slot Classification <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Joint_Intent_and_Slot_Classification.ipynb>`_
|
||||
* - NLP
|
||||
- GLUE Benchmark
|
||||
- `GLUE Benchmark <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/GLUE_Benchmark.ipynb>`_
|
||||
* - NLP
|
||||
- Punctuation and Capitalization
|
||||
- `Punctuation and Capitalization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Punctuation_and_Capitalization.ipynb>`_
|
||||
* - NLP
|
||||
- Named Entity Recognition - BioMegatron
|
||||
- `Named Entity Recognition - BioMegatron <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Token_Classification-BioMegatron.ipynb>`_
|
||||
* - NLP
|
||||
- Relation Extraction - BioMegatron
|
||||
- `Relation Extraction - BioMegatron <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Relation_Extraction-BioMegatron.ipynb>`_
|
||||
* - TTS
|
||||
- Speech Synthesis
|
||||
- `TTS Inference <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/tts/1_TTS_inference.ipynb>`_
|
||||
* - TTS
|
||||
- Speech Synthesis
|
||||
- `Tacotron2 Training <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/tts/2_TTS_Tacotron2_Training.ipynb>`_
|
||||
* - Tools
|
||||
- CTC Segmentation
|
||||
- `CTC Segmentation <https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/tools/CTC_Segmentation_Tutorial.ipynb>`_
|
||||
* - Text Processing
|
||||
- Text Normalization for TTS
|
||||
- `Text Normalization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/text_processing/Text_Normalization.ipynb>`_
|
||||
* - Text Processing
|
||||
- Inverse Text Normalization for ASR
|
||||
- `Inverse Text Normalization <https://colab.research.google.com/github/NVIDIA/NeMo/blob/main/tutorials/text_processing/Inverse_Text_Normalization.ipynb>`_
|
||||
@@ -0,0 +1,20 @@
|
||||
Dataset Creation Tool Based on CTC-Segmentation
|
||||
===============================================
|
||||
|
||||
This tool provides functionality to align long audio files and the corresponding transcripts into shorter fragments
|
||||
that are suitable for an Automatic Speech Recognition (ASR) model training.
|
||||
|
||||
More details could be found in `NeMo/tutorials/tools/CTC_Segmentation_Tutorial.ipynb <https://github.com/NVIDIA/NeMo/blob/main/tutorials/tools/CTC_Segmentation_Tutorial.ipynb>`__ (can be executed with `Google's Colab <https://colab.research.google.com/notebooks/intro.ipynb>`_).
|
||||
|
||||
The tool is based on the `CTC-Segmentation <https://github.com/lumaku/ctc-segmentation>`__ package and
|
||||
`CTC-Segmentation of Large Corpora for German End-to-end Speech Recognition
|
||||
<https://arxiv.org/abs/2007.09127>`__ :cite:`tools-kurzinger2020ctc`
|
||||
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: tools_all.bib
|
||||
:style: plain
|
||||
:labelprefix: TOOLS
|
||||
:keyprefix: tools-
|
||||
@@ -0,0 +1,14 @@
|
||||
Tools
|
||||
=====
|
||||
|
||||
NeMo provides a set of tools useful for developing Automatic Speech Recognitions (ASR) and Text-to-Speech (TTS) models: \
|
||||
`https://github.com/NVIDIA/NeMo/tree/main/tools <https://github.com/NVIDIA/NeMo/tree/main/tools>`__ .
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
ctc_segmentation
|
||||
speech_data_explorer
|
||||
text_processing_deployment
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
Speech Data Explorer
|
||||
====================
|
||||
|
||||
.. note::
|
||||
|
||||
The tool could be found under `NeMo/tools/speech_data_explorer <https://github.com/NVIDIA/NeMo/tree/main/tools/speech_data_explorer>`__.
|
||||
|
||||
`Dash <https://plotly.com/dash/>`__-based tool for interactive exploration of ASR/TTS datasets.
|
||||
|
||||
Features:
|
||||
|
||||
* dataset's statistics (alphabet, vocabulary, duration-based histograms)
|
||||
* navigation across dataset (sorting, filtering)
|
||||
* inspection of individual utterances (waveform, spectrogram, audio player)
|
||||
* errors' analysis (Word Error Rate, Character Error Rate, Word Match Rate, Mean Word Accuracy, diff)
|
||||
|
||||
Please make sure that requirements are installed. Then run:
|
||||
|
||||
.. code::
|
||||
|
||||
python data_explorer.py path_to_manifest.json
|
||||
|
||||
|
||||
JSON manifest file should contain the following fields:
|
||||
|
||||
* `audio_filepath` (path to audio file)
|
||||
* `duration` (duration of the audio file in seconds)
|
||||
* `text` (reference transcript)
|
||||
|
||||
Errors' analysis requires "pred_text" (ASR transcript) for all utterances.
|
||||
|
||||
Any additional field will be parsed and displayed in 'Samples' tab.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
NeMo Text Processing Deployment
|
||||
===============================================
|
||||
|
||||
This tool deploys :doc:`NeMo Inverse Text Normalization (ITN) <../nemo_text_processing/inverse_text_normalization>` and :doc:`NeMo Text Normalization (TN) <../nemo_text_processing/text_normalization>` for production :cite:`tools-itn_deploy-zhang2021nemo`.
|
||||
It uses `Sparrowhawk <https://github.com/google/sparrowhawk>`_ :cite:`tools-itn_deploy-sparrowhawk` -- an open-source version of Google Kestrel :cite:`tools-itn_deploy-ebden2015kestrel`.
|
||||
|
||||
Requirements
|
||||
------------------------
|
||||
|
||||
:doc:`nemo_text_processing <../nemo_text_processing/intro>` package
|
||||
|
||||
|
||||
Usage
|
||||
------------
|
||||
|
||||
Starts docker container with production backend with plugged in grammars. This is entry point script.
|
||||
|
||||
Arguments:
|
||||
^^^^^^^^^
|
||||
* ``GRAMMARS`` - ``tn_grammars`` or ``itn_grammars`` to export either TN or ITN grammars from :doc:`nemo_text_processing <../nemo_text_processing/intro>`.
|
||||
* ``INPUT_CASE`` - ``cased`` or ``lower_cased`` (lower_cased is supported only in TN grammars).
|
||||
* ``MODE`` - choose ``test`` to run test on the grammars inside the container.
|
||||
|
||||
For example:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# to export ITN grammars
|
||||
bash export_grammar.sh --GRAMMARS=itn_grammars
|
||||
|
||||
# to export and test TN grammars
|
||||
bash export_grammar.sh --GRAMMARS=itn_grammars --INPUT_CASE=cased --MODE=test
|
||||
|
||||
This script runs the following steps in sequence:
|
||||
|
||||
Exports grammar `ClassifyFst` and `VerbalizeFst` from :doc:`nemo_text_processing <../nemo_text_processing/intro>` to `OUTPUT_DIR/classify/tokenize_and_classify.far` and `OUTPUT_DIR/verbalize/verbalize.far` respectively.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python pynini_export.py <--output_dir OUTPUT_DIR> <--grammars GRAMMARS> <--input_case INPUT_CASE>
|
||||
|
||||
Builds C++ production backend docker
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
bash docker/build.sh
|
||||
|
||||
|
||||
Plugs in grammars into production backend by mounting grammar directory `classify/` and `verbalize/` with sparrowhawk grammar directory inside docker. Returns docker prompt
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
# to launch container with the exported grammars
|
||||
bash docker/launch.sh
|
||||
|
||||
# to launch container with the exported grammars and run tests on TN grammars
|
||||
bash docker/launch.sh test_tn_grammars
|
||||
|
||||
# to launch container with the exported grammars and run tests on ITN grammars
|
||||
bash docker/launch.sh test_itn_grammars
|
||||
|
||||
|
||||
Runs TN or ITN in docker container:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
echo "two dollars fifty" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto
|
||||
|
||||
This returns $2.50 for ITN.
|
||||
|
||||
References
|
||||
----------
|
||||
|
||||
.. bibliography:: tools_all.bib
|
||||
:style: plain
|
||||
:labelprefix: TOOLS-ITN_DEPLOY
|
||||
:keyprefix: tools-itn_deploy-
|
||||
@@ -0,0 +1,49 @@
|
||||
@inproceedings{kurzinger2020ctc,
|
||||
title={CTC-segmentation of large corpora for german end-to-end speech recognition},
|
||||
author={K{\"u}rzinger, Ludwig and Winkelbauer, Dominik and Li, Lujun and Watzel, Tobias and Rigoll, Gerhard},
|
||||
booktitle={International Conference on Speech and Computer},
|
||||
pages={267--278},
|
||||
year={2020},
|
||||
organization={Springer}
|
||||
}
|
||||
|
||||
@article{ebden2015kestrel,
|
||||
title={The Kestrel TTS text normalization system},
|
||||
author={Ebden, Peter and Sproat, Richard},
|
||||
journal={Natural Language Engineering},
|
||||
volume={21},
|
||||
number={3},
|
||||
pages={333},
|
||||
year={2015},
|
||||
publisher={Cambridge University Press}
|
||||
}
|
||||
|
||||
@article{sproat2016rnn,
|
||||
title={RNN approaches to text normalization: A challenge},
|
||||
author={Sproat, Richard and Jaitly, Navdeep},
|
||||
journal={arXiv preprint arXiv:1611.00068},
|
||||
year={2016}
|
||||
}
|
||||
|
||||
@book{taylor2009text,
|
||||
title={Text-to-speech synthesis},
|
||||
author={Taylor, Paul},
|
||||
year={2009},
|
||||
publisher={Cambridge university press}
|
||||
}
|
||||
|
||||
@misc{zhang2021nemo,
|
||||
title={NeMo Inverse Text Normalization: From Development To Production},
|
||||
author={Yang Zhang and Evelina Bakhturina and Kyle Gorman and Boris Ginsburg},
|
||||
year={2021},
|
||||
eprint={2104.05055},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL}
|
||||
}
|
||||
|
||||
@inproceedings{sparrowhawk,
|
||||
title = {TTS for Low Resource Languages: A Bangla Synthesizer},
|
||||
author = {Alexander Gutkin and Linne Ha and Martin Jansche and Knot Pipatsrisawat and Richard Sproat},
|
||||
booktitle = {10th Language Resources and Evaluation Conference},
|
||||
year = {2016},
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
NeMo TTS Collection API
|
||||
=======================
|
||||
|
||||
TTS Base Classes
|
||||
----------------
|
||||
|
||||
The classes below are the base of the TTS pipeline.
|
||||
To read more about them, see the `Base Classes <./intro.html#Base Classes>`__ section of the intro page.
|
||||
|
||||
.. autoclass:: nemo.collections.tts.models.base.SpectrogramGenerator
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.models.base.Vocoder
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.models.base.TextToWaveform
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
TTS Datasets
|
||||
------------
|
||||
|
||||
.. autoclass:: nemo.collections.tts.data.datalayers.AudioDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.data.datalayers.MelAudioDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.data.datalayers.SplicedAudioDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.data.datalayers.NoisySpecsDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.asr.data.audio_to_text.FastPitchDataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
|
||||
.. autoclass:: nemo.collections.tts.data.datalayers.FastSpeech2Dataset
|
||||
:show-inheritance:
|
||||
:members:
|
||||
@@ -0,0 +1,187 @@
|
||||
TTS Datasets
|
||||
============
|
||||
|
||||
Most TTS scripts work out-of-the-box with the LJ Speech dataset.
|
||||
However, some models may require supplementary data for training and validation.
|
||||
The following sections contain details of required data formats and instructions for running preprocessing scripts for
|
||||
such models.
|
||||
|
||||
|
||||
FastSpeech 2
|
||||
------------
|
||||
|
||||
The FastSpeech 2 model converts from phonemes to Mel Spectrograms and predicts phoneme durations, pitches, and
|
||||
energies.
|
||||
Therefore, in addition to the manifest and audio samples, it requires some supplementary files and data:
|
||||
|
||||
* A **mappings file** for converting from words to phonemes and phonemes to indices
|
||||
* **Phoneme durations** (for each audio sample)
|
||||
* **Pitch per frame** (for each training audio sample)
|
||||
* **Energy per frame** (for each training audio sample)
|
||||
* (Optional) **Ignore list** for filtering out samples with OOV words
|
||||
|
||||
The ``FastSpeech2Dataset`` uses the manifest format shared with all other NeMo speech tasks.
|
||||
Each line of the manifest should describe one sample, and should be in the following format:
|
||||
|
||||
.. code::
|
||||
|
||||
{"audio_filepath": "/data/wavs/audio.wav", "text": "an unused transcription", "duration": 23.147}
|
||||
|
||||
See the documentation on :ref:`Preparing Custom ASR Data` for more details on the NeMo speech manifest format.
|
||||
|
||||
.. note::
|
||||
The ``FastSpeech2Dataset`` ignores the ``"text"`` field of the manifest, since the model reads phoneme indices from
|
||||
the supplementary duration files instead.
|
||||
|
||||
The **mappings file** should be a JSON-formatted file that contains dictionaries ``word2phones`` and ``phone2idx``.
|
||||
``word2phones`` is a mapping that converts from your vocabulary to phonemes.
|
||||
For example, one entry in ``word2phones`` might be ``"nemo": ["N", "EH1", "M", "OW0"]``.
|
||||
``phone2idx`` is an (arbitrary) mapping that converts each phoneme to an index for prediction.
|
||||
It is also used to determine the ``pad_id``, which is set to the length of ``phone2idx``.
|
||||
|
||||
Together, these two dictionaries are also used for inference, as an input sentence is first normalized, then converted
|
||||
to phonemes using ``word2phones``, and then converted from phonemes to indices using ``phone2idx``.
|
||||
|
||||
For each audio sample, there should be a corresponding **phoneme durations** file that contains ``token_duration``
|
||||
(duration per phoneme in frames) and ``text_encoded`` (index of each phoneme corresponding to ``phone2idx``).
|
||||
|
||||
For each sample, there should also be a **pitch file** that contains the pitches (F0) per frame, and an **energy file**
|
||||
that contains the energies (L2-norm of STFT frame amplitudes) per frame.
|
||||
|
||||
The **ignore list** is a pickled list of file base names (with no extension) that tells the ``FastSpeech2Dataset``
|
||||
which audio samples to discard.
|
||||
You should use this to exclude samples from your manifest that contain OOV words.
|
||||
|
||||
|
||||
Directory Structure
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
While the mappings file and ignore list paths are passed in to the ``FastSpeech2Dataset`` directly, the
|
||||
``FastSpeech2Dataset`` infers the path to the supplementary files corresponding to each audio sample based on the
|
||||
audio samples' paths found in the manifest.
|
||||
For each sample, the paths to the corresponding duration, pitch, and energy files are inferred by replacing ``wavs/``
|
||||
with ``phoneme_durations/``, ``pitches/``, and ``energies``, and swapping out the file extension (``.wav``) with
|
||||
``.pt``, ``.npy``, and ``.npy`` respectively.
|
||||
|
||||
For example, given manifest audio path ``/data/LJSpeech/wavs/LJ001-0001.wav``, the inferred duration and phonemes file
|
||||
path would be ``/data/LJSpeech/phoneme_durations/LJ001-0001.pt``.
|
||||
|
||||
Your directory structure should therefore look something like this:
|
||||
|
||||
.. code::
|
||||
|
||||
data/
|
||||
├── manifest.json
|
||||
├── mappings.json
|
||||
├── ignore_file.pkl
|
||||
├── energies/
|
||||
│ ├── basename_1.npy
|
||||
│ ├── basename_2.npy
|
||||
│ ├── ...
|
||||
│ └── basename_n.npy
|
||||
├── phoneme_durations/
|
||||
│ ├── basename_1.pt
|
||||
│ ├── basename_2.pt
|
||||
│ ├── ...
|
||||
│ └── basename_n.pt
|
||||
├── pitches/
|
||||
│ ├── basename_1.npy
|
||||
│ ├── basename_2.npy
|
||||
│ ├── ...
|
||||
│ └── basename_n.npy
|
||||
└── wavs/
|
||||
├── basename_1.wav
|
||||
├── basename_2.wav
|
||||
├── ...
|
||||
└── basename_n.wav
|
||||
|
||||
|
||||
LJ Speech Dataset
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
NeMo comes with scripts for preprocessing the LJ Speech dataset for use with FastSpeech 2, which can be found in the
|
||||
``<nemo_base>/scripts/dataset_processing/ljspeech/`` directory.
|
||||
These scripts assume that you have downloaded the LJ Speech dataset and extracted it to ``<ljspeech_base>``.
|
||||
|
||||
They perform the following dataset preparation steps:
|
||||
|
||||
* Create manifest files and ``.txt`` files with normalized transcripts
|
||||
* Extract pitches
|
||||
* Extract energies
|
||||
* Downloads the `CMU Pronouncing Dictionary <http://www.speech.cs.cmu.edu/cgi-bin/cmudict>`_ to convert normalized
|
||||
transcripts to phonemes
|
||||
* Uses the `Montreal Forced Aligner <https://montreal-forced-aligner.readthedocs.io/en/latest/index.html>`_ (MFA) to
|
||||
perform alignment
|
||||
* Calculates durations based on the alignment
|
||||
* Generates the mappings file and ignore file for the dataset
|
||||
|
||||
To run the scripts, follow the steps below.
|
||||
|
||||
#. Download and extract the `LJ Speech dataset <https://keithito.com/LJ-Speech-Dataset/>`_ to ``<ljspeech_base>``.
|
||||
|
||||
#. Create the manifest files and normalized text files (for MFA to discover later) by running:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python create_manifests_and_textfiles.py --ljspeech_base=<ljspeech_base>
|
||||
|
||||
#. Extract pitches and energies from the audio files:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
python extract_ljspeech_energy_pitch.py --ljspeech_base=<ljspeech_base>
|
||||
|
||||
This will write the extracted pitches and energies to ``<ljspeech_base>/pitches/`` and
|
||||
``<ljspeech_base>/energies/``.
|
||||
|
||||
#. Run the phoneme extraction and duration calculation script.
|
||||
This script will set up a Conda environment to download and install MFA and its dependencies, so make sure that you
|
||||
have Anaconda or Miniconda before running the script.
|
||||
|
||||
Also note that MFA does sometimes have trouble finding OpenBlas, so you may have to manually install it with the
|
||||
command ``sudo apt-get install libopenblas-dev``.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
./extract_ljspeech_phonemes_and_durs.sh <ljspeech_base>
|
||||
|
||||
This script takes additional options ``--skip_env_setup`` if you have already set up the ``aligner`` environment,
|
||||
and ``--g2p_dict=<dict_path>`` if you have already downloaded CMU dict or another G2P dictionary you prefer.
|
||||
|
||||
The alignment step will take a while to run, so be prepared to wait upwards of an hour.
|
||||
|
||||
In addition to alignments and durations, this script will create the mappings and ignore files as well.
|
||||
It will also generate some intermediate files that you are free to delete, as you only need the files listed above.
|
||||
|
||||
When the following steps are finished, your ``<ljspeech_base>`` directory will look like this.
|
||||
(Starred files and directories indicate files from intermediate steps that are safe to remove, but that you may want
|
||||
to keep for bookkeeping purposes.)
|
||||
|
||||
.. code::
|
||||
|
||||
<ljspeech_base>/
|
||||
├── alignments/ *
|
||||
├── cmudict.dict *
|
||||
├── energies/
|
||||
│ ├── LJ001-0001.npy
|
||||
│ └── ...
|
||||
├── ljs_audio_text_test_filelist.txt *
|
||||
├── ljs_audio_text_train_filelist.txt *
|
||||
├── ljs_audio_text_val_filelist.txt *
|
||||
├── ljspeech_test.json
|
||||
├── ljspeech_train.json
|
||||
├── ljspeech_val.json
|
||||
├── mappings.json
|
||||
├── metadata.csv
|
||||
├── phoneme_durations/
|
||||
│ ├── LJ001-0001.pt
|
||||
│ └── ...
|
||||
├── pitches/
|
||||
│ ├── LJ001-0001.npy
|
||||
│ └── ...
|
||||
├── README
|
||||
├── uncommented_cmudict.dict *
|
||||
├── wavs/
|
||||
│ ├── LJ001-0001.wav
|
||||
│ └── ...
|
||||
└── wavs_to_ignore.pkl
|
||||
@@ -0,0 +1,153 @@
|
||||
Speech Synthesis (TTS)
|
||||
======================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 4
|
||||
:hidden:
|
||||
|
||||
datasets
|
||||
api
|
||||
|
||||
Speech Synthesis or Text-to-Speech (TTS) involves turning text into human speech. The NeMo TTS collection currently
|
||||
supports two pipelines for TTS:
|
||||
|
||||
1) The two stage pipeline. First, a model is used to generate a mel spectrogram from text. Second, a model is used
|
||||
to generate audio from a mel spectrogram.
|
||||
2) The "end-to-end" approach that uses one model to generate audio straight from text.
|
||||
|
||||
Quick Start:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import soundfile as sf
|
||||
from nemo.collections.tts.models.base import SpectrogramGenerator, Vocoder
|
||||
|
||||
# Download and load the pretrained tacotron2 model
|
||||
spec_generator = SpectrogramGenerator.from_pretrained("tts_en_tacotron2")
|
||||
# Download and load the pretrained waveglow model
|
||||
vocoder = Vocoder.from_pretrained("tts_waveglow_88m")
|
||||
|
||||
# All spectrogram generators start by parsing raw strings to a tokenized version of the string
|
||||
parsed = spec_generator.parse("You can type your sentence here to get nemo to produce speech.")
|
||||
# They then take the tokenized string and produce a spectrogram
|
||||
spectrogram = spec_generator.generate_spectrogram(tokens=parsed)
|
||||
# Finally, a vocoder converts the spectrogram to audio
|
||||
audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram)
|
||||
|
||||
# Save the audio to disk in a file called speech.wav
|
||||
# Note vocoder return a batch of audio. In this example, we just take the first and only sample.
|
||||
sf.write("speech.wav", audio.to('cpu').detach().numpy()[0], 22050)
|
||||
|
||||
.. note::
|
||||
|
||||
For an interactive version of the quick start above, refer to the TTS inference notebook that can be found on the
|
||||
github readme.
|
||||
|
||||
Available Models
|
||||
################
|
||||
|
||||
NeMo supports a variety of models that can be used for TTS. For beginners, we recommend starting with the Tacotron2 +
|
||||
WaveGlow combination. Then we suggest trying using (TalkNet or FastPitch) + HiFiGAN if you want to continue exploring
|
||||
the two stage pipeline, or the FastPitch_HifiGan_E2E model for the end-to-end pipeline.
|
||||
|
||||
.. list-table:: *TTS Models*
|
||||
:widths: 5 5 10 25
|
||||
:header-rows: 1
|
||||
|
||||
* - Model
|
||||
- Base Class
|
||||
- Pretrained Checkpoint
|
||||
- Description
|
||||
* - Tacotron2
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_tacotron2
|
||||
- LSTM encoder decoder based model that generates spectrograms
|
||||
* - GlowTTS
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_glowtts
|
||||
- Glow-based spectrogram generator
|
||||
* - FastSpeech 2
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_fastspeech_2
|
||||
- Non-autoregressive transformer-based spectrogram generator that predicts duration, energy, and pitch
|
||||
* - FastPitch
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- (Coming soon)
|
||||
- Non-autoregressive transformer-based spectrogram generator that predicts duration and pitch
|
||||
* - TalkNet
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_en_talknet
|
||||
- Non-autoregressive convolution-based spectrogram generator
|
||||
* - WaveGlow
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_waveglow_88m
|
||||
- Glow-based vocoder
|
||||
* - SqueezeWave
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_squeezewave
|
||||
- Glow-based vocoder based on WaveGlow but with fewer parameters
|
||||
* - UniGlow
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_uniglow
|
||||
- Glow-based vocoder based on WaveGlow but shares 1 set of parameters across all flow steps
|
||||
* - MelGAN
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_melgan
|
||||
- GAN-based vocoder
|
||||
* - HiFiGAN
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
- https://ngc.nvidia.com/catalog/models/nvidia:nemo:tts_hifigan
|
||||
- GAN-based vocoder
|
||||
* - FastPitch_HifiGan_E2E
|
||||
- :class:`TextToWaveform<nemo.collections.tts.models.base.TextToWaveform>`
|
||||
- (Coming soon)
|
||||
- End-to-end model based on composing FastPitch and HiFiGAN
|
||||
* - FastSpeech2_HifiGan_E2E
|
||||
- :class:`TextToWaveform<nemo.collections.tts.models.base.TextToWaveform>`
|
||||
- (Coming soon)
|
||||
- End-to-end model based on composing FastSpeech2 and HiFiGAN
|
||||
|
||||
Base Classes
|
||||
############
|
||||
|
||||
Two Stage Pipeline
|
||||
******************
|
||||
|
||||
NeMo TTS has two base classes corresponding to the two stage pipeline:
|
||||
|
||||
- :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>`
|
||||
- :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>`
|
||||
|
||||
The :class:`SpectrogramGenerator<nemo.collections.tts.models.base.SpectrogramGenerator>` class has two important
|
||||
functions: :py:meth:`parse<nemo.collections.tts.models.base.SpectrogramGenerator.parse>` which
|
||||
accepts raw python strings and returns a torch.tensor that represents tokenized text ready to pass to, and
|
||||
:py:meth:`generate_spectrogram<nemo.collections.tts.models.base.SpectrogramGenerator.generate_spectrogram>` which
|
||||
accepts a batch of tokenized text and returns a torch.tensor that represents a batch of spectrograms
|
||||
|
||||
The :class:`Vocoder<nemo.collections.tts.models.base.Vocoder>` class has one important functions
|
||||
:py:meth:`convert_spectrogram_to_audio<nemo.collections.tts.models.base.Vocoder.convert_spectrogram_to_audio>` which
|
||||
accepts a batch of spectrograms and returns a torch.tensor that represents a batch of raw audio.
|
||||
|
||||
End-to-End Pipeline
|
||||
*******************
|
||||
|
||||
Correspondingly, NeMo TTS has one base class for the end-to-end pipeline:
|
||||
|
||||
- :class:`TextToWaveform<nemo.collections.tts.models.base.TextToWaveform>`
|
||||
|
||||
Similarly to the SpectrogramGenerator, :class:`TextToWaveform<nemo.collections.tts.models.base.TextToWaveform>`
|
||||
implements two functions: :py:meth:`parse<nemo.collections.tts.models.base.TextToWaveform.parse>` which
|
||||
accepts raw python strings and returns a torch.tensor that represents tokenized text ready to pass to, and
|
||||
:py:meth:`generate_spectrogram<nemo.collections.tts.models.base.TextToWaveform.convert_text_to_waveform>` which
|
||||
accepts a batch of tokenized text and returns a torch.tensor that represents a batch of audio.
|
||||
|
||||
TTS Training
|
||||
############
|
||||
|
||||
Training of TTS models can be done using the scripts inside the NeMo ``examples/tts folders``. The majority of the TTS
|
||||
YAML configurations should work out of the box with the LJSpeech dataset. If you want to train on other data, it is
|
||||
recommended that you walk through the Tacotron 2 Training notebook. Please pay special attention to the sample rate and
|
||||
FFT parameters for new data.
|
||||
|
||||
Some models, like FastSpeech 2, require supplementary data such as phoneme durations, pitches, and energies.
|
||||
For more information about how to preprocess datasets for such models, see the `TTS Datasets <./datasets.html>`__ page.
|
||||
Reference in New Issue
Block a user