mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-23 08:26:06 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
pretrained_model: null # pretrained QAModel model from list_available_models()
|
||||
do_training: true # training mode, for testing change to false
|
||||
trainer:
|
||||
gpus: 8 # number of GPUs (0 for CPU), or list of the GPUs to use e.g. [0, 1]
|
||||
num_nodes: 1
|
||||
max_epochs: 3
|
||||
max_steps: null # precedence over max_epochs
|
||||
accumulate_grad_batches: 1 # accumulates grads every k batches
|
||||
gradient_clip_val: 1.0
|
||||
amp_level: O1 # O1/O2 for mixed precision
|
||||
precision: 16 # Should be set to 16 for O1 and O2 to enable the AMP.
|
||||
accelerator: ddp
|
||||
log_every_n_steps: 5 # Interval of logging.
|
||||
val_check_interval: 1.0 # Set to 0.25 to check 4 times per epoch, or an int for number of iterations
|
||||
resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.
|
||||
num_sanity_val_steps: 0 # number of steps to perform validation steps for sanity check the validation process before starting the training, setting to 0 disables it
|
||||
checkpoint_callback: False # Provided by exp_manager
|
||||
logger: False # Provided by exp_manager
|
||||
|
||||
model:
|
||||
nemo_path: null # filename to save the model and associated artifacts to .nemo file
|
||||
tokenizer:
|
||||
tokenizer_name: ${model.language_model.pretrained_model_name} # or sentencepiece
|
||||
vocab_file: null # path to vocab file
|
||||
tokenizer_model: null # only used if tokenizer is sentencepiece
|
||||
special_tokens: null
|
||||
|
||||
language_model:
|
||||
pretrained_model_name: bert-base-cased
|
||||
lm_checkpoint: null
|
||||
config_file: null # json file, precedence over config
|
||||
config: null
|
||||
|
||||
encoder:
|
||||
dropout: 0.1
|
||||
|
||||
dataset:
|
||||
max_seq_length: 128 # the maximum length BERT supports is 512
|
||||
max_num_cat_slot: 6 # maximum number of different categorical slots per service in dataset
|
||||
max_num_noncat_slot: 12 # maximum number of different non-categorical slots per service in dataset
|
||||
max_value_per_cat_slot: 12 # maximum number of different categorical slot values per service in dataset
|
||||
max_num_intent: 4 # maximum number of different intents per service in dataset
|
||||
task_name: sgd_single_domain # or from [sgd_all, sgd_all_single, sgd_multi_domain, debug_sample]
|
||||
state_tracker: nemotracker # or baseline
|
||||
data_dir: ???
|
||||
dialogues_example_dir: ??? # cache location to store or load preprocessed data from. To use cache, specify `use_cache`
|
||||
subsample: false # balances negative and positive training examples for improved performance
|
||||
use_cache: false # uses a cache to store the processed dataset, you may use it for large datasets for speed up
|
||||
use_fuzzy_match: true # Whether to use fuzzy string matching when comparing non-categorical slot values. Should be set to False when conducting multiwoz style evaluation.
|
||||
joint_acc_across_turn: false # Whether to compute joint goal accuracy across turn instead of across service. Should be set to True when conducting multiwoz style evaluation.
|
||||
|
||||
train_ds:
|
||||
ds_item: "train"
|
||||
batch_size: 128
|
||||
shuffle: true
|
||||
num_workers: 3
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
validation_ds:
|
||||
ds_item: ["dev"] # ["dev", "test"]
|
||||
batch_size: 8
|
||||
shuffle: false
|
||||
num_workers: 3
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
test_ds:
|
||||
ds_item: ["test"]
|
||||
batch_size: 8
|
||||
shuffle: false
|
||||
num_workers: 3
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 1e-4
|
||||
# optimizer arguments
|
||||
betas: [0.9, 0.999]
|
||||
weight_decay: 0.01
|
||||
|
||||
# scheduler setup
|
||||
sched:
|
||||
name: PolynomialDecayAnnealing
|
||||
# Scheduler params
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.02
|
||||
last_epoch: -1
|
||||
# pytorch lightning args
|
||||
monitor: val_loss
|
||||
reduce_on_plateau: false
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null # exp_dir for your experiment, if None, defaults to "./nemo_experiments"
|
||||
name: "SGDQA" # The name of your model
|
||||
create_tensorboard_logger: True # Whether you want exp_manger to create a tb logger
|
||||
create_checkpoint_callback: True # Whether you want exp_manager to create a modelcheckpoint callback
|
||||
resume_if_exists: false
|
||||
resume_ignore_no_checkpoint: false
|
||||
@@ -0,0 +1,145 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright 2019 The Google Research Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script contains an example of how to train and test the NeMo SGD-QA Model (https://arxiv.org/abs/2105.08049).
|
||||
The SGD-QA model is a fast multi-pass schema-guided state-tracking model, that is trained on the Google schema-guided state tracking dataset (https://arxiv.org/abs/1909.05855).
|
||||
The model takes dialogue as input and outputs the dialogue state, which includes slot-value pairs.
|
||||
The model consists of two components: a neural natural language understanding model (NLU), and a rule-based state tracker.
|
||||
The NLU takes in a dialogue turn and different schema (entity) information options and outputs their match score. The state tracker takes the highest rated entities and composes
|
||||
the dialogue state across turns.
|
||||
|
||||
***Data format***
|
||||
The SGD-QA requires a JSON schema and dialogue files for each dataset split.
|
||||
In the following we will show an example for a service entry in the schema file.
|
||||
* service_name
|
||||
* description
|
||||
* slots
|
||||
* name
|
||||
* description
|
||||
* is_categorical
|
||||
* possible values
|
||||
* intents
|
||||
* name
|
||||
* description
|
||||
* required_slots (not used)
|
||||
* is_transactional (not used)
|
||||
* optional_slots (not used)
|
||||
* result_slots (not used)
|
||||
|
||||
|
||||
In the following we will show an example for a dialogue. More information at https://arxiv.org/abs/1909.05855
|
||||
* dialogue_id
|
||||
* services
|
||||
* turns
|
||||
* frames
|
||||
* actions
|
||||
* act
|
||||
* slot
|
||||
* values
|
||||
* service
|
||||
* slots
|
||||
* exclusive_end
|
||||
* slot
|
||||
* start
|
||||
* state
|
||||
* active_intent
|
||||
* requeste_slots
|
||||
* slot_values
|
||||
* speaker - [USER, SYSTEM]
|
||||
* utterance
|
||||
|
||||
|
||||
***Downloading the dataset***
|
||||
# git clone https://github.com/google-research-datasets/dstc8-schema-guided-dialogue.git
|
||||
|
||||
***Setting the configs***
|
||||
The model and the PT trainer are defined in a config file that declares multiple important sections.
|
||||
The most important ones are:
|
||||
model: All arguments that are related to the Model - language model, SGD-QA encoder and decoder, loss, optimizer,
|
||||
schedulers, and datasets/data loaders.
|
||||
trainer: Any argument to be passed to PyTorch Lightning including number of epochs, number of GPUs,
|
||||
precision level, etc.
|
||||
|
||||
This script uses the `/examples/nlp/dialogue_state_tracking/conf/sgdqa_config.yaml` config file
|
||||
by default. You may update the config file from the file directly. The other option is to set another config file via command-line arguments by `--config-name=CONFIG_FILE_PATH'.
|
||||
|
||||
|
||||
***Model Training***
|
||||
# python sgd_qa.py
|
||||
do_training=True
|
||||
model.dataset.data_dir=<DATA_DIR_WITH_JSON_DATA>
|
||||
model.dataset.dialogues_example_dir=<DATA_DIR_WITH_PREPROCESSED_DATA>
|
||||
model.validation_ds.ds_item=<LIST_OF_SPLITS>
|
||||
trainer.max_epochs=<NUM_EPOCHS>
|
||||
trainer.gpus=[<CHANGE_TO_GPU_YOU_WANT_TO_USE>]
|
||||
|
||||
|
||||
***Model Evaluation***
|
||||
# python sgd_qa.py
|
||||
do_training=False
|
||||
model.test_ds.ds_item=<LIST_OF_SPLITS>
|
||||
|
||||
To load a pretrained checkpoint from the cloud prior to training (e.g. for fine-tuning) or evaluation you can set cfg.from_pretrained=<MODEL_NAME>. You can find all pretrained model names by using
|
||||
SGDQAModel.list_available_models(). To load a local checkpoint use model.restore_from(<PATH_TO_CHECKPOINT>)
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from nemo.collections.nlp.models.dialogue_state_tracking.sgdqa_model import SGDQAModel
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils import logging
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
@hydra_runner(config_path="conf", config_name="sgdqa_config")
|
||||
def main(cfg: DictConfig) -> None:
|
||||
logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
|
||||
trainer = pl.Trainer(**cfg.trainer)
|
||||
exp_manager(trainer, cfg.get("exp_manager", None))
|
||||
|
||||
if cfg.pretrained_model or (cfg.model.nemo_path and os.path.exists(cfg.model.nemo_path)):
|
||||
if cfg.pretrained_model:
|
||||
logging.info(f'Loading pretrained model {cfg.pretrained_model}')
|
||||
model = SGDQAModel.from_pretrained(cfg.pretrained_model)
|
||||
else:
|
||||
logging.info(f'Restoring model from {cfg.model.nemo_path}')
|
||||
model = SGDQAModel.restore_from(cfg.model.nemo_path)
|
||||
if cfg.do_training:
|
||||
model.setup_training_data(train_data_config=cfg.model.train_ds)
|
||||
model.setup_multiple_validation_data(val_data_config=cfg.model.validation_ds)
|
||||
else:
|
||||
logging.info(f'Config: {OmegaConf.to_yaml(cfg)}')
|
||||
model = SGDQAModel(cfg.model, trainer=trainer)
|
||||
|
||||
if cfg.do_training:
|
||||
trainer.fit(model)
|
||||
if cfg.model.nemo_path:
|
||||
model.save_to(cfg.model.nemo_path)
|
||||
|
||||
if hasattr(cfg.model, 'test_ds') and cfg.model.test_ds.ds_item is not None:
|
||||
gpu = 1 if cfg.trainer.gpus != 0 else 0
|
||||
trainer = pl.Trainer(gpus=gpu)
|
||||
model.setup_multiple_test_data(test_data_config=cfg.model.test_ds)
|
||||
if model.prepare_test(trainer):
|
||||
trainer.test(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user