Introduce code from Rust monorepo. This is a mess

2026-10-22 08:16:08 +00:00 · 2022-02-28 05:43:54 -05:00
parent 921fb49f9a
commit ba326bfcfd
1058 changed files with 192192 additions and 0 deletions
@@ -0,0 +1,135 @@
+# Question Answering with SQUAD
+name: &name QA
+
+pretrained_model: null # pretrained QAModel model from list_available_models()
+do_training: true # training mode, for testing change to false
+trainer:
+  gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices
+  num_nodes: 1
+  max_epochs: 2 # the number of training epochs
+  max_steps: null # precedence over max_epochs
+  accumulate_grad_batches: 1 # accumulates grads every k batches
+  precision: 16 # 16 to use AMP
+  amp_level: O1 # O1 or O2 if using AMP
+  accelerator: ddp
+  gradient_clip_val: 0.0
+  val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
+  checkpoint_callback: false # provided by exp_manager
+  logger: false # provided by exp_manager
+  num_sanity_val_steps: 0
+  log_every_n_steps: 1  # Interval of logging.
+
+model:
+  nemo_path: null # exported .nemo path
+  dataset:
+    version_2_with_negative: false
+    # If true, the examples contain some that do not have an answer.
+    doc_stride: 128
+    # When splitting up a long document into chunks,
+    # how much stride to take between chunks.
+    max_query_length: 64
+    # The maximum number of tokens for the question.
+    # Questions longer than this will be truncated to
+    # this length.
+    max_seq_length: 384
+    # The maximum total input sequence length after
+    # WordPiece tokenization. Sequences longer than this
+    # will be truncated, and sequences shorter than this
+    # will be padded.
+    max_answer_length: 30
+    # The maximum length of an answer that can be
+    # generated. This is needed because the start
+    # and end predictions are not conditioned
+    # on one another.
+    null_score_diff_threshold: 0.0
+    # If null_score - best_non_null is greater than the threshold predict null.
+    n_best_size: 20
+    # The total number of n-best predictions to generate at testing.
+    use_cache: true
+    do_lower_case: true
+    # if true does lower case
+
+    num_workers:  2
+    pin_memory: false
+    drop_last: false
+
+  train_ds:
+    file: null # .json file
+    batch_size: 24 # per GPU
+    shuffle: true
+    num_samples: -1
+    # Default values for the following params are retrieved from dataset config section, but you may override them
+    num_workers: ${model.dataset.num_workers}
+    drop_last: ${model.dataset.drop_last}
+    pin_memory: ${model.dataset.pin_memory}
+
+  validation_ds:
+    file: null # .json file
+    batch_size: 24 # per GPU
+    shuffle: false
+    num_samples: -1    
+    # Default values for the following params are retrieved from dataset config section, but you may override them
+    num_workers: ${model.dataset.num_workers}
+    drop_last: ${model.dataset.drop_last}
+    pin_memory: ${model.dataset.pin_memory}
+
+  test_ds:
+    file: null # .json file
+    batch_size: 24 # per GPU
+    shuffle: false
+    num_samples: -1
+    # Default values for the following params are retrieved from dataset config section, but you may override them
+    num_workers: ${model.dataset.num_workers}
+    drop_last: ${model.dataset.drop_last}
+    pin_memory: ${model.dataset.pin_memory}
+    
+  tokenizer:
+    tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
+    vocab_file: null # path to vocab file
+    tokenizer_model: null # only used if tokenizer is sentencepiece
+    special_tokens: null # expand the following to a dictionary if special tokens need to be added.
+    #  only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently.
+
+  language_model:
+    pretrained_model_name: bert-base-uncased # BERT-like model name
+    lm_checkpoint: null
+    config_file: null # json file, precedence over config
+    config: null # if specified initializes model from scratch
+
+  token_classifier:
+    num_layers: 1
+    dropout: 0.0
+    num_classes: 2
+    activation: relu
+    log_softmax: false
+    use_transformer_init: true
+
+
+  optim:
+    name: adamw
+    lr: 3e-5
+    weight_decay: 0.0
+    sched:
+      name: SquareRootAnnealing
+
+      # pytorch lightning args
+      monitor: val_loss
+      reduce_on_plateau: false
+
+      # scheduler config override
+      warmup_steps: null
+      warmup_ratio: 0.0
+      last_epoch: -1
+
+exp_manager:
+  exp_dir: null # where to store logs and checkpoints
+  name: *name # name of experiment
+  create_tensorboard_logger: True
+  create_checkpoint_callback: True
+
+hydra:
+  run:
+    dir: .
+  job_logging:
+    root:
+      handlers: null
@@ -0,0 +1,68 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import urllib.request
+
+from nemo.utils import logging
+
+
+class SquadDownloader:
+    def __init__(self, save_path):
+        self.save_path = save_path + '/squad'
+
+        if not os.path.exists(self.save_path):
+            os.makedirs(self.save_path)
+
+        if not os.path.exists(self.save_path + '/v1.1'):
+            os.makedirs(self.save_path + '/v1.1')
+
+        if not os.path.exists(self.save_path + '/v2.0'):
+            os.makedirs(self.save_path + '/v2.0')
+
+        self.download_urls = {
+            'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/train-v1.1.json': 'v1.1/train-v1.1.json',
+            'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/dev-v1.1.json': 'v1.1/dev-v1.1.json',
+            'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/train-v2.0.json': 'v2.0/train-v2.0.json',
+            'https://rajpurkar.github.io/SQuAD-explorer' '/dataset/dev-v2.0.json': 'v2.0/dev-v2.0.json',
+        }
+
+    def download(self):
+        for item in self.download_urls:
+            url = item
+            file = self.download_urls[item]
+
+            logging.info('Downloading: %s', url)
+            if os.path.isfile(self.save_path + '/' + file):
+                logging.info('** Download file already exists, skipping download')
+            else:
+                response = urllib.request.urlopen(url)
+                with open(self.save_path + '/' + file, "wb") as handle:
+                    handle.write(response.read())
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Download Squad')
+    parser.add_argument(
+        '--destDir',
+        type=str,
+        required=False,
+        help='directory to store data',
+        default=os.path.split(os.path.abspath(__file__))[0],
+    )
+    args = parser.parse_args()
+    logging.info(args.destDir)
+    squad_dl = SquadDownloader(args.destDir)
+    squad_dl.download()
@@ -0,0 +1,134 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This script contains an example on how to train, evaluate and perform inference with the question answering model.
+The QAModel in NeMo supports extractive question answering problems for data in the SQuAD (https://rajpurkar.github.io/SQuAD-explorer/) format.
+
+***Data format***
+The QAModel requires a JSON file for each dataset split. 
+In the following we will show example for a training file. Each title has one or multiple paragraph entries, each consisting of the text - "context", and question-answer entries. Each question-answer entry has:
+* a question
+* a globally unique id
+* a boolean flag "is_impossible" which shows if the question is answerable or not
+* in case the question is answerable one answer entry, which contains the text span and its starting character index in the context. If not answerable, the "answers" list is empty
+
+The evaluation file follows the above format except for it can provide more than one answers to the same question. 
+The inference file follows the above format except for it does not require the "answers" and "is_impossible" keywords.
+
+
+***Downloading the dataset***
+Run ./NeMo/examples/nlp/question_answering/get_squad.py to download the SQuAD dataset:
+
+#   python get_squad.py --destDir=<PATH_TO_DATA>
+
+***Setting the configs***
+The model and the PT trainer are defined in a config file which declares multiple important sections.
+The most important ones are:
+    model: All arguments that are related to the Model - language model, tokenizer, token classifier, optimizer,
+            schedulers, and datasets/data loaders.
+    trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs,
+            precision level, etc.
+
+This script uses the `/examples/nlp/question_answering/conf/question_answering_squad_config.yaml` config file
+by default. You may update the config file from the file directly. The other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'.
+
+
+***Model Training***
+# python question_answering_squad.py
+    model.train_ds.file=<TRAIN_JSON_FILE>
+    model.validation_ds=<VAL_JSON_FILE>
+    trainer.max_epochs=<NUM_EPOCHS>
+    trainer.gpus=[<CHANGE_TO_GPU_YOU_WANT_TO_USE>]
+
+
+***Model Evaluation***
+Set `do_training=False` in the script and run:
+
+#   python question_answering_squad.py
+    model.test_file=<TEST_JSON_FILE>
+
+To load a pretrained checkpoint from cloud prior to training (e.g. for fine-tuning) or evaluation you can set cfg.from_pretrained=<MODEL_NAME>,
+e.g. MODEL_NAME='BERTBaseUncasedSQuADv1.1'. You can find all pretrained model names by using 
+QAModel.list_available_models(). To load a local checkpoint use qa_model.restore_from(<PATH_TO_CHECKPOINT>)
+
+
+***Model Inference***
+For inference use 
+    qa_model.inference(
+        file=<INFERENCE_JSON_FILE>,
+        batch_size=<BATCH_SIZE>,
+        output_nbest_file=<OPTIONAL_OUTPUT_FILE_FOR_NBEST_LIST>,
+        output_prediction_file=<OPTIONAL_OUTPUT_FILE_FOR_PREDICTION>
+    )
+
+More details on how to use this script can be found in
+./NeMo/tutorials/nlp/Question_Answering_Squad.ipynb
+"""
+
+import os
+
+import pytorch_lightning as pl
+from omegaconf import DictConfig, OmegaConf
+
+from nemo.collections.nlp.models.question_answering.qa_model import QAModel
+from nemo.core.config import hydra_runner
+from nemo.utils import logging
+from nemo.utils.exp_manager import exp_manager
+
+
+@hydra_runner(config_path="conf", config_name="question_answering_squad_config")
+def main(cfg: DictConfig) -> None:
+    logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
+    trainer = pl.Trainer(**cfg.trainer)
+    exp_dir = exp_manager(trainer, cfg.get("exp_manager", None))
+
+    if not cfg.pretrained_model:
+        logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
+        model = QAModel(cfg.model, trainer=trainer)
+    else:
+        logging.info(f'Loading pretrained model {cfg.pretrained_model}')
+        model = QAModel.from_pretrained(cfg.pretrained_model)
+        if cfg.do_training:
+            model.setup_training_data(train_data_config=cfg.model.train_ds)
+            model.setup_validation_data(val_data_config=cfg.model.validation_ds)
+
+    if cfg.do_training:
+        trainer.fit(model)
+        if cfg.model.nemo_path:
+            model.save_to(cfg.model.nemo_path)
+
+    if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.file is not None:
+        trainer.test(model)
+
+    # change to path if you want results to be written to file e.g. os.path.join(exp_dir, "output_nbest_file.txt")
+    output_nbest_file = None
+    # change to path if you want results to be written to file e.g.  os.path.join(exp_dir, "output_prediction_file.txt")
+    output_prediction_file = None
+    inference_samples = 5  # for test purposes. To use entire inference dataset set to -1
+    all_preds, all_nbests = model.inference(
+        file=cfg.model.validation_ds.file,
+        batch_size=1,
+        num_samples=inference_samples,
+        output_nbest_file=output_nbest_file,
+        output_prediction_file=output_prediction_file,
+    )
+
+    for _, item in all_preds.items():
+        print(f"question: {item[0]} answer: {item[1]}")
+
+
+if __name__ == '__main__':
+    main()