Introduce code from Rust monorepo. This is a mess

tts/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_preprocessed_config.yaml

@@ -0,0 +1,80 @@
+# BERT Pretraining from Preprocessed (tokenized) data
+name: &name PretrainingBERTFromPreprocessed
+trainer:
+  gpus: 8 # the number of gpus, 0 for CPU, or list with gpu indices
+  num_nodes: 1
+  max_steps: 2285714 # precedence over max_epochs
+  num_sanity_val_steps: 0 # needed for bert pretraining from preproc
+  replace_sampler_ddp: false # needed for bert pretraining from preproc
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  precision: 16 # 16 to use AMP
+  amp_level: O1 # O1 or O2 if using AMP
+  accelerator: ddp
+  gradient_clip_val: 1.0
+  log_every_n_steps: 1
+  val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
+  checkpoint_callback: false # provided by exp_manager
+  logger: false # provided by exp_manager
+
+model:
+  nemo_path: null # exported .nemo path
+  only_mlm_loss: true # only use masked language model without next sentence prediction
+  num_tok_classification_layers: 1 # number of token classification head output layers
+  num_seq_classification_layers: 2 # number of sequence classification head output layers
+
+
+  language_model:
+    pretrained_model_name: bert-base-uncased # huggingface model name
+    lm_checkpoint: null
+    config:
+      attention_probs_dropout_prob: 0.1
+      hidden_act: gelu
+      hidden_dropout_prob: 0.1
+      hidden_size: 768
+      initializer_range: 0.02
+      intermediate_size: 3072
+      max_position_embeddings: 512
+      num_attention_heads: 12
+      num_hidden_layers: 12
+      type_vocab_size: 2
+      vocab_size: 30522
+    config_file: null # json file, precedence over config
+
+  tokenizer: null
+
+  train_ds:
+    data_file: null # path to hdf5 file (or directory)
+    max_predictions_per_seq: 80
+    batch_size: 16
+    shuffle: true
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  optim:
+    name: adamw
+    lr: 0.4375e-4
+    weight_decay: 0.01
+
+    sched:
+      name: SquareRootAnnealing
+      warmup_steps: null
+      warmup_ratio: 0.01
+      min_lr: 0.0
+      last_epoch: -1
+
+
+exp_manager:
+  exp_dir: null # where to store logs and checkpoints
+  name: *name # name of experiment
+  create_tensorboard_logger: True
+  create_checkpoint_callback: True
+
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null

tts/NeMo/examples/nlp/language_modeling/conf/bert_pretraining_from_text_config.yaml

@@ -0,0 +1,109 @@
+# BERT Pretraining from Text
+name: &name PretrainingBERTFromText
+trainer:
+  gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices
+  num_nodes: 1
+  max_epochs: 2 # the number of training epochs
+  max_steps: null # precedence over max_epochs
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  precision: 16 # 16 to use AMP
+  amp_level: O1 # O1 or O2 if using AMP
+  accelerator: ddp
+  gradient_clip_val: 0.0
+  log_every_n_steps: 1
+  val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
+  checkpoint_callback: false # provided by exp_manager
+  logger: false # provided by exp_manager
+
+model:
+  nemo_path: null # exported .nemo path
+  only_mlm_loss: false # only use masked language model without next sentence prediction
+  num_tok_classification_layers: 1 # number of token classification head output layers
+  num_seq_classification_layers: 2 # number of sequence classification head output layers
+  max_seq_length: 128
+  # The maximum total input sequence length after tokenization. Sequences longer than this
+  # will be truncated, and sequences shorter than this will be padded.
+  mask_prob: 0.15
+  # Probability of masking a token in the input text during data processing.
+  short_seq_prob: 0.1
+  # Probability of having a sequence shorter than the maximum sequence length `max_seq_length` in data processing.",
+
+  language_model:
+    pretrained_model_name: bert-base-uncased
+    lm_checkpoint: null
+    config:
+      attention_probs_dropout_prob: 0.1
+      hidden_act: gelu
+      hidden_dropout_prob: 0.1
+      hidden_size: 768
+      initializer_range: 0.02
+      intermediate_size: 3072
+      max_position_embeddings: 512
+      num_attention_heads: 12
+      num_hidden_layers: 12
+      type_vocab_size: 2
+      vocab_size: 30522
+    config_file: null # json file, precedence over config
+
+  tokenizer:
+    tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
+    vocab_file: null # path to vocab file
+    tokenizer_model: null # tokenizer model for sentencepiece
+    special_tokens: # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently.
+        unk_token: '[UNK]'
+        sep_token: '[SEP]'
+        pad_token: '[PAD]'
+        bos_token: '[CLS]'
+        mask_token: '[MASK]'
+        eos_token: '[SEP]'
+        cls_token: '[CLS]'
+
+  train_ds:
+    data_file: null # path to data file
+    max_seq_length: ${model.max_seq_length}
+    mask_prob: ${model.mask_prob}
+    short_seq_prob: ${model.short_seq_prob}
+    batch_size: 16 # per GPU
+    shuffle: true
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  validation_ds:
+    data_file: null # path to data file
+    max_seq_length: ${model.max_seq_length}
+    mask_prob: ${model.mask_prob}
+    short_seq_prob: ${model.short_seq_prob}
+    batch_size: 16 # per GPU
+    shuffle: false
+    num_samples: -1
+    num_workers: 2
+    drop_last: false
+    pin_memory: false
+
+  optim:
+    name: adamw
+    lr: 3e-5
+    weight_decay: 0.0
+
+    sched:
+      name: CosineAnnealing
+      warmup_steps: null
+      warmup_ratio: 0.1
+      min_lr: 0.0
+      last_epoch: -1
+
+
+exp_manager:
+  exp_dir: null # where to store logs and checkpoints
+  name: *name # name of experiment
+  create_tensorboard_logger: True
+  create_checkpoint_callback: True
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null

tts/NeMo/examples/nlp/language_modeling/conf/transformer_lm_config.yaml

@@ -0,0 +1,96 @@
+name: TransformerLanguageModel
+do_training: True # set to False if only preprocessing data
+
+model:
+  label_smoothing: 0.0
+  preproc_out_dir: null # path to store data preprocessing outputs
+  
+  train_ds:
+    use_tarred_dataset: true
+    tar_files: ??? # path to tarred files
+    metadata_file: ??? # metadata for tarred dataset
+    shard_strategy: scatter
+    tokens_in_batch: 4096
+    tar_shuffle_n: 256
+    
+  validation_ds:
+    file_name: ??? # path to file with validation data
+    tokens_in_batch: 512
+    clean: false
+    shuffle: false
+    num_samples: -1
+    drop_last: false
+    pin_memory: false
+    num_workers: 8
+    
+  test_ds:
+    file_name: ??? # path to file with test data
+    tokens_in_batch: 512
+    clean: false
+    shuffle: false
+    num_samples: -1
+    drop_last: false
+    pin_memory: false
+    num_workers: 8
+    
+  optim:
+    name: adam
+    lr: 0.001
+    betas:
+      - 0.9
+      - 0.98
+    weight_decay: 0.0
+    sched:
+      name: InverseSquareRootAnnealing
+      min_lr: 0.0
+      last_epoch: -1
+      warmup_ratio: 0.1
+
+  tokenizer:
+    tokenizer_name: yttm
+    tokenizer_model: ???
+    vocab_file: null
+    special_tokens: null
+    training_sample_size: null # valid for sentencepiece tokenizer
+
+  encoder:
+    library: nemo
+    model_name: null
+    pretrained: false
+    max_sequence_length: 512
+    num_token_types: 0
+    embedding_dropout: 0.1
+    learn_positional_encodings: false
+    hidden_size: 512
+    num_layers: 6
+    inner_size: 2048
+    num_attention_heads: 8
+    ffn_dropout: 0.1
+    attn_score_dropout: 0.1
+    attn_layer_dropout: 0.1
+    hidden_act: relu
+    mask_future: true
+    pre_ln: false
+
+  head:
+    num_layers: 1
+    activation: relu
+    log_softmax: true
+    dropout: 0.0
+    use_transformer_init: true
+
+trainer:
+  gpus: 4
+  num_nodes: 1
+  max_epochs: 200
+  amp_level: O2 # O1/O2 for mixed precision
+  precision: 16 # Should be set to 16 for O1 and O2, default is 16 as PT ignores it when am_level is O0
+  accelerator: ddp
+  checkpoint_callback: False
+  logger: False
+  log_every_n_steps: 50  # Interval of logging.
+  check_val_every_n_epoch: 1
+
+exp_manager:
+  name: TransformerLM
+  files_to_copy: []