Introduce code from Rust monorepo. This is a mess

tts/NeMo/examples/nlp/glue_benchmark/glue_benchmark.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+## Tasks
+This script works with all GLUE Benchmark tasks, more details about the GLUE Benchmark could be found at
+https://gluebenchmark.com/
+
+More details on how to use this script could be found in tutorials/nlp/GLUE_Benchmark.ipynb
+
+## Model Training
+
+To train GLUEModel with the default config file, run:
+    python glue_benchmark.py \
+    model.dataset.data_dir=<PATH_TO_DATA_DIR>  \
+    model.task_name=TASK_NAME \
+    trainer.max_epochs=<NUM_EPOCHS> \
+    trainer.gpus="[<CHANGE_TO_GPU_YOU_WANT_TO_USE>]
+
+Supported task names:
+["cola", "sst-2", "mrpc", "sts-b", "qqp", "mnli", "qnli", "rte", "wnli"]
+Note, MNLI task includes both matched and mismatched dev sets
+"""
+
+import os
+
+import pytorch_lightning as pl
+from omegaconf import DictConfig, OmegaConf
+
+from nemo.collections.nlp.models import GLUEModel
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+from nemo.utils.exp_manager import exp_manager
+
+
+@hydra_runner(config_name="glue_benchmark_config")
+def main(cfg: DictConfig) -> None:
+    logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
+    trainer = pl.Trainer(**cfg.trainer)
+    exp_manager_cfg = cfg.get("exp_manager", None)
+
+    if exp_manager_cfg:
+        exp_manager_cfg.name = cfg.model.task_name
+        logging.info(f'Setting task_name to {exp_manager_cfg.name} in exp_manager')
+    exp_manager(trainer, exp_manager_cfg)
+
+    if cfg.model.nemo_path and os.path.exists(cfg.model.nemo_path):
+        model = GLUEModel.restore_from(cfg.model.nemo_path)
+        logging.info(f'Restoring model from {cfg.model.nemo_path}')
+        model.update_data_dir(data_dir=cfg.model.dataset.data_dir)
+        model.setup_training_data()
+        model.setup_multiple_validation_data()
+        trainer.fit(model)
+    else:
+        model = GLUEModel(cfg.model, trainer=trainer)
+        trainer.fit(model)
+        if cfg.model.nemo_path:
+            model.save_to(cfg.model.nemo_path)
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/examples/nlp/glue_benchmark/glue_benchmark_config.yaml

@@ -0,0 +1,82 @@
+# GLUE Benchmark with pre-trained BERT models
+supported_tasks: &supported_tasks ['cola', 'sst-2', 'mrpc', 'sts-b', 'qqp', 'mnli', 'qnli', 'rte', 'wnli']
+
+trainer:
+  gpus: 1 # the number of gpus, 0 for CPU
+  num_nodes: 1
+  max_epochs: 3
+  max_steps: null # precedence over max_epochs
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  amp_level: O0 # O1/O2 for mixed precision
+  precision: 16
+  accelerator: ddp
+  checkpoint_callback: False  # Provided by exp_manager
+  logger: False  # Provided by exp_manager
+
+model:
+  task_name: &task_name mrpc # choose from: ["cola", "sst-2", "mrpc", "sts-b", "qqp", "mnli", "qnli", "rte", "wnli"] GLUE task name, MNLI includes both matched and mismatched dev sets
+  supported_tasks: *supported_tasks
+  output_dir: null # dir to write write predictions
+  nemo_path: null # filename to save the model and associated artifacts to .nemo file
+  dataset:
+    data_dir: ??? # /path/to/data
+    max_seq_length: 128
+    use_cache: true
+
+    # shared across dataloaders:
+    num_workers:  2
+    pin_memory: false
+    drop_last: false
+
+  train_ds:
+    ds_item: 'train.tsv'
+    shuffle: true
+    num_samples: -1
+    batch_size: 32
+
+  validation_ds:
+    ds_item: 'dev.tsv' # for MNLI 'dev_matched.tsv' and 'dev_mismatched.tsv' will de used
+    shuffle: false
+    num_samples: -1
+    batch_size: 32
+
+  tokenizer:
+      tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece
+      vocab_file: null # path to vocab file
+      tokenizer_model: null # only used if tokenizer is sentencepiece
+      special_tokens: null # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently.
+
+  language_model:
+    pretrained_model_name: bert-base-uncased
+    lm_checkpoint: null
+    config_file: null # json file, precedence over config
+    config: null
+
+  optim:
+    name: adam
+    lr: 5e-5
+    weight_decay: 0.00
+
+    sched:
+      name: WarmupAnnealing
+      # Scheduler params
+      warmup_steps: null
+      warmup_ratio: 0.1
+      last_epoch: -1
+
+      # pytorch lightning args
+      monitor: val_loss
+      reduce_on_plateau: false
+
+exp_manager:
+  exp_dir: null  # exp_dir for your experiment, if None, defaults to "./NeMo_experiments"
+  name: *task_name # The name of your model
+  create_tensorboard_logger: True  # Whether you want exp_manger to create a tb logger
+  create_checkpoint_callback: True  # Whether you want exp_manager to create a modelcheckpoint callback
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null