Introduce code from Rust monorepo. This is a mess

tts/NeMo/examples/nlp/language_modeling/bert_pretraining.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pytorch_lightning as pl
+from omegaconf import DictConfig, OmegaConf
+from pytorch_lightning.plugins import DDPPlugin
+
+from nemo.collections.nlp.models.language_modeling import BERTLMModel
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+from nemo.utils.exp_manager import exp_manager
+
+
+@hydra_runner(config_path="conf", config_name="bert_pretraining_from_text_config")
+def main(cfg: DictConfig) -> None:
+    logging.info(f'Config:\n {OmegaConf.to_yaml(cfg)}')
+    trainer = pl.Trainer(plugins=[DDPPlugin(find_unused_parameters=True)], **cfg.trainer)
+    exp_manager(trainer, cfg.get("exp_manager", None))
+    bert_model = BERTLMModel(cfg.model, trainer=trainer)
+    trainer.fit(bert_model)
+    if cfg.model.nemo_path:
+        bert_model.save_to(cfg.model.nemo_path)
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml

@@ -0,0 +1,80 @@
+# BERT Pretraining from Preprocessed (tokenized) data
+name: &name PretrainingBERTFromPreprocessed
+trainer:
+  gpus: 8 # the number of gpus, 0 for CPU, or list with gpu indices
+  num_nodes: 1
+  max_steps: 2285714 # precedence over max_epochs
+  num_sanity_val_steps: 0 # needed for bert pretraining from preproc
+  replace_sampler_ddp: false # needed for bert pretraining from preproc
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  precision: 16 # 16 to use AMP
+  amp_level: O1 # O1 or O2 if using AMP
+  accelerator: ddp
+  gradient_clip_val: 1.0
+  log_every_n_steps: 1
+  val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
+  checkpoint_callback: false # provided by exp_manager
+  logger: false # provided by exp_manager
+
+model:
+  nemo_path: null # exported .nemo path
+  only_mlm_loss: true # only use masked language model without next sentence prediction
+  num_tok_classification_layers: 1 # number of token classification head output layers
+  num_seq_classification_layers: 2 # number of sequence classification head output layers
+
+
+  language_model:
+    pretrained_model_name: bert-base-uncased # huggingface model name
+    lm_checkpoint: null
+    config:
+      attention_probs_dropout_prob: 0.1
+      hidden_act: gelu
+      hidden_dropout_prob: 0.1
+      hidden_size: 768
+      initializer_range: 0.02
+      intermediate_size: 3072
+      max_position_embeddings: 512
+      num_attention_heads: 12
+      num_hidden_layers: 12
+      type_vocab_size: 2
+      vocab_size: 30522
+    config_file: null # json file, precedence over config
+
+  tokenizer: null
+
+  train_ds:
+    data_file: null # path to hdf5 file (or directory)
+    max_predictions_per_seq: 80
+    batch_size: 16
+    shuffle: true
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  optim:
+    name: adamw
+    lr: 0.4375e-4
+    weight_decay: 0.01
+
+    sched:
+      name: SquareRootAnnealing
+      warmup_steps: null
+      warmup_ratio: 0.01
+      min_lr: 0.0
+      last_epoch: -1
+
+
+exp_manager:
+  exp_dir: null # where to store logs and checkpoints
+  name: *name # name of experiment
+  create_tensorboard_logger: True
+  create_checkpoint_callback: True
+
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null

tts/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml

@@ -0,0 +1,109 @@
+# BERT Pretraining from Text
+name: &name PretrainingBERTFromText
+trainer:
+  gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices
+  num_nodes: 1
+  max_epochs: 2 # the number of training epochs
+  max_steps: null # precedence over max_epochs
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  precision: 16 # 16 to use AMP
+  amp_level: O1 # O1 or O2 if using AMP
+  accelerator: ddp
+  gradient_clip_val: 0.0
+  log_every_n_steps: 1
+  val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
+  checkpoint_callback: false # provided by exp_manager
+  logger: false # provided by exp_manager
+
+model:
+  nemo_path: null # exported .nemo path
+  only_mlm_loss: false # only use masked language model without next sentence prediction
+  num_tok_classification_layers: 1 # number of token classification head output layers
+  num_seq_classification_layers: 2 # number of sequence classification head output layers
+  max_seq_length: 128
+  # The maximum total input sequence length after tokenization. Sequences longer than this
+  # will be truncated, and sequences shorter than this will be padded.
+  mask_prob: 0.15
+  # Probability of masking a token in the input text during data processing.
+  short_seq_prob: 0.1
+  # Probability of having a sequence shorter than the maximum sequence length `max_seq_length` in data processing.",
+
+  language_model:
+    pretrained_model_name: bert-base-uncased
+    lm_checkpoint: null
+    config:
+      attention_probs_dropout_prob: 0.1
+      hidden_act: gelu
+      hidden_dropout_prob: 0.1
+      hidden_size: 768
+      initializer_range: 0.02
+      intermediate_size: 3072
+      max_position_embeddings: 512
+      num_attention_heads: 12
+      num_hidden_layers: 12
+      type_vocab_size: 2
+      vocab_size: 30522
+    config_file: null # json file, precedence over config
+
+  tokenizer:
+    tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
+    vocab_file: null # path to vocab file
+    tokenizer_model: null # tokenizer model for sentencepiece
+    special_tokens: # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently.
+        unk_token: '[UNK]'
+        sep_token: '[SEP]'
+        pad_token: '[PAD]'
+        bos_token: '[CLS]'
+        mask_token: '[MASK]'
+        eos_token: '[SEP]'
+        cls_token: '[CLS]'
+
+  train_ds:
+    data_file: null # path to data file
+    max_seq_length: ${model.max_seq_length}
+    mask_prob: ${model.mask_prob}
+    short_seq_prob: ${model.short_seq_prob}
+    batch_size: 16 # per GPU
+    shuffle: true
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  validation_ds:
+    data_file: null # path to data file
+    max_seq_length: ${model.max_seq_length}
+    mask_prob: ${model.mask_prob}
+    short_seq_prob: ${model.short_seq_prob}
+    batch_size: 16 # per GPU
+    shuffle: false
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  optim:
+    name: adamw
+    lr: 3e-5
+    weight_decay: 0.0
+
+    sched:
+      name: CosineAnnealing
+      warmup_steps: null
+      warmup_ratio: 0.1
+      min_lr: 0.0
+      last_epoch: -1
+
+
+exp_manager:
+  exp_dir: null # where to store logs and checkpoints
+  name: *name # name of experiment
+  create_tensorboard_logger: True
+  create_checkpoint_callback: True
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null

tts/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml

@@ -0,0 +1,96 @@
+name: TransformerLanguageModel
+do_training: True # set to False if only preprocessing data
+
+model:
+  label_smoothing: 0.0
+  preproc_out_dir: null # path to store data preprocessing outputs
+  
+  train_ds:
+    use_tarred_dataset: true
+    tar_files: ??? # path to tarred files
+    metadata_file: ??? # metadata for tarred dataset
+    shard_strategy: scatter
+    tokens_in_batch: 4096
+    tar_shuffle_n: 256
+    
+  validation_ds:
+    file_name: ??? # path to file with validation data
+    tokens_in_batch: 512
+    clean: false
+    shuffle: false
+    num_samples: -1
+    drop_last: false
+    pin_memory: false
+    num_workers: 8
+    
+  test_ds:
+    file_name: ??? # path to file with test data
+    tokens_in_batch: 512
+    clean: false
+    shuffle: false
+    num_samples: -1
+    drop_last: false
+    pin_memory: false
+    num_workers: 8
+    
+  optim:
+    name: adam
+    lr: 0.001
+    betas:
+      - 0.9
+      - 0.98
+    weight_decay: 0.0
+    sched:
+      name: InverseSquareRootAnnealing
+      min_lr: 0.0
+      last_epoch: -1
+      warmup_ratio: 0.1
+
+  tokenizer:
+    tokenizer_name: yttm
+    tokenizer_model: ???
+    vocab_file: null
+    special_tokens: null
+    training_sample_size: null # valid for sentencepiece tokenizer
+
+  encoder:
+    library: nemo
+    model_name: null
+    pretrained: false
+    max_sequence_length: 512
+    num_token_types: 0
+    embedding_dropout: 0.1
+    learn_positional_encodings: false
+    hidden_size: 512
+    num_layers: 6
+    inner_size: 2048
+    num_attention_heads: 8
+    ffn_dropout: 0.1
+    attn_score_dropout: 0.1
+    attn_layer_dropout: 0.1
+    hidden_act: relu
+    mask_future: true
+    pre_ln: false
+
+  head:
+    num_layers: 1
+    activation: relu
+    log_softmax: true
+    dropout: 0.0
+    use_transformer_init: true
+
+trainer:
+  gpus: 4
+  num_nodes: 1
+  max_epochs: 200
+  amp_level: O2 # O1/O2 for mixed precision
+  precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0
+  accelerator: ddp
+  checkpoint_callback: False
+  logger: False
+  log_every_n_steps: 50  # Interval of logging.
+  check_val_every_n_epoch: 1
+
+exp_manager:
+  name: TransformerLM
+  files_to_copy: []

tts/NeMo/examples/nlp/language_modeling/convert_weights_to_nemo1.0.py

@@ -0,0 +1,61 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Converts BERT NeMo0.* checkpoints to NeMo1.0 format.
+"""
+
+from argparse import ArgumentParser
+
+import torch
+
+parser = ArgumentParser()
+parser.add_argument("--bert_encoder", required=True, help="path to BERT encoder, e.g. /../BERT-STEP-2285714.pt")
+parser.add_argument(
+    "--bert_token_classifier",
+    required=True,
+    help="path to BERT token classifier, e.g. /../BertTokenClassifier-STEP-2285714.pt",
+)
+parser.add_argument(
+    "--bert_sequence_classifier",
+    required=False,
+    default=None,
+    help="path to BERT sequence classifier, e.g /../SequenceClassifier-STEP-2285714.pt",
+)
+parser.add_argument(
+    "--output_path", required=False, default="converted_model.pt", help="output path to newly converted model"
+)
+args = parser.parse_args()
+
+bert_in = torch.load(args.bert_encoder)
+tok_in = torch.load(args.bert_token_classifier)
+if args.bert_sequence_classifier:
+    seq_in = torch.load(args.bert_sequence_classifier)
+
+new_dict = {}
+new_model = {"state_dict": new_dict}
+for k in bert_in:
+    new_name = k.replace("bert.", "bert_model.")
+    new_dict[new_name] = bert_in[k]
+
+for k in tok_in:
+    new_name = "mlm_classifier." + k
+    new_dict[new_name] = tok_in[k]
+
+if args.bert_sequence_classifier:
+    for k in seq_in:
+        new_name = "nsp_classifier." + k
+        new_dict[new_name] = seq_in[k]
+
+torch.save(new_model, args.output_path)

tts/NeMo/examples/nlp/language_modeling/get_wkt2.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+"""
+This file is adapted from
+https://github.com/salesforce/awd-lstm-lm/blob/master/getdata.sh
+Copyright by the AWD LSTM authors.
+"""
+DATA_DIR=$1
+echo "- Downloading WikiText-2"
+
+wget --continue -P $DATA_DIR https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip
+unzip -q $DATA_DIR/wikitext-2-v1.zip -d $DATA_DIR
+cd $DATA_DIR/wikitext-2
+mv wiki.train.tokens train.txt
+sed -i -e "s/<unk>/[UNK]/g" train.txt
+mv wiki.valid.tokens valid.txt
+sed -i -e "s/<unk>/[UNK]/g" valid.txt
+mv wiki.test.tokens test.txt
+sed -i -e "s/<unk>/[UNK]/g" test.txt
+cd ..
+rm wikitext-2-v1.zip
+
+echo "- WikiText-2 saved at $DATA_DIR/wikitext-2"

tts/NeMo/examples/nlp/language_modeling/transformer_lm.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pytorch_lightning as pl
+from omegaconf import DictConfig, OmegaConf
+
+from nemo.collections.nlp.models.language_modeling import TransformerLMModel
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+from nemo.utils.exp_manager import exp_manager
+
+
+@hydra_runner(config_path="conf", config_name="transformer_lm_config")
+def main(cfg: DictConfig) -> None:
+    logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
+    trainer = pl.Trainer(**cfg.trainer)
+    exp_manager(trainer, cfg.get("exp_manager", None))
+    transformer_lm = TransformerLMModel(cfg.model, trainer=trainer)
+    trainer.fit(transformer_lm)
+
+
+if __name__ == '__main__':
+    main()