mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-20 08:04:35 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,105 @@
|
||||
# Text2Sparql with BART
|
||||
|
||||
name: &name Text2Sparql
|
||||
|
||||
trainer:
|
||||
gpus: 1 # the number of gpus, 0 for CPU, or list with gpu indices
|
||||
num_nodes: 1
|
||||
max_epochs: 2 # the number of training epochs
|
||||
max_steps: null # precedence over max_epochs
|
||||
accumulate_grad_batches: 1 # accumulates grads every k batches
|
||||
accelerator: ddp
|
||||
gradient_clip_val: 0.0
|
||||
log_every_n_steps: 1
|
||||
val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
|
||||
checkpoint_callback: false # provided by exp_manager
|
||||
logger: false # provided by exp_manager
|
||||
|
||||
model:
|
||||
nemo_path: null # exported .nemo path
|
||||
max_seq_length: 150
|
||||
batch_size: 16
|
||||
convert_labels: true # true if Bart, false otherwise (converts pad_id to -100 for masked loss)
|
||||
data_dir: null
|
||||
|
||||
language_model:
|
||||
pretrained_model_name: facebook/bart-base # huggingface end-to-end model name
|
||||
pretrained_encoder_model_name: null # huggingface encoder model name
|
||||
pretrained_decoder_model_name: null # huggingface decoder model name
|
||||
lm_checkpoint: null
|
||||
config: null
|
||||
config_file: null # json file, precedence over config
|
||||
|
||||
encoder_tokenizer:
|
||||
tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
|
||||
vocab_file: null # path to vocab file
|
||||
tokenizer_model: null # tokenizer model for sentencepiece
|
||||
special_tokens: null
|
||||
add_special_tokens: true
|
||||
|
||||
decoder_tokenizer:
|
||||
tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
|
||||
vocab_file: null # path to vocab file
|
||||
tokenizer_model: null # tokenizer model for sentencepiece
|
||||
special_tokens: null
|
||||
add_special_tokens: true
|
||||
|
||||
train_ds:
|
||||
filepath: ${model.data_dir}/train.tsv # path to data file
|
||||
shuffle: true
|
||||
num_samples: -1
|
||||
num_workers: 2
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
validation_ds:
|
||||
filepath: ${model.data_dir}/test_easy.tsv # path to data file
|
||||
shuffle: false
|
||||
num_samples: -1
|
||||
num_workers: 2
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
test_ds:
|
||||
filepath: ${model.data_dir}/test_hard.tsv # path to data file
|
||||
shuffle: false
|
||||
num_samples: -1
|
||||
num_workers: 2
|
||||
drop_last: false
|
||||
pin_memory: false
|
||||
|
||||
optim:
|
||||
name: adamw
|
||||
lr: 4e-5
|
||||
weight_decay: 0.0
|
||||
|
||||
sched:
|
||||
name: CosineAnnealing
|
||||
warmup_steps: null
|
||||
warmup_ratio: 0.06
|
||||
min_lr: 0.0
|
||||
last_epoch: -1
|
||||
|
||||
generate:
|
||||
max_length: ${model.max_seq_length}
|
||||
num_beams: 1
|
||||
length_penalty: 2.0
|
||||
early_stopping: true
|
||||
repetition_penalty: 1.0
|
||||
do_sample: false
|
||||
top_k: null
|
||||
top_p: null
|
||||
num_return_sequences: 1
|
||||
|
||||
exp_manager:
|
||||
exp_dir: null # where to store logs and checkpoints
|
||||
name: *name # name of experiment
|
||||
create_tensorboard_logger: True
|
||||
create_checkpoint_callback: True
|
||||
|
||||
hydra:
|
||||
run:
|
||||
dir: .
|
||||
job_logging:
|
||||
root:
|
||||
handlers: null
|
||||
@@ -0,0 +1,134 @@
|
||||
# Copyright (c) 2020, MeetKai Inc. All rights reserved.
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script downloads Text2Sparql data and processes it into NeMo's neural machine translation dataset format.
|
||||
|
||||
Text2Sparql data consists of 3 files which are saved to source_data_dir:
|
||||
- train_queries_v3.tsv
|
||||
- test_easy_queries_v3.tsv
|
||||
- test_hard_queries_v3.tsv
|
||||
|
||||
After processing, the script saves them to the target_data_dir as:
|
||||
- train.tsv
|
||||
- test_easy.tsv
|
||||
- test_hard.tsv
|
||||
|
||||
|
||||
You may run it with:
|
||||
|
||||
python import_datasets \
|
||||
--source_data_dir ./text2sparql_src \
|
||||
--target_data_dir ./text2sparql_tgt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import os
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from nemo.collections.nlp.data.data_utils.data_preprocessing import MODE_EXISTS_TMP, if_exist
|
||||
from nemo.utils import logging
|
||||
|
||||
base_url = "https://m.meetkai.com/public_datasets/knowledge/"
|
||||
prefix_map = {
|
||||
"train_queries_v3.tsv": "train.tsv",
|
||||
"test_easy_queries_v3.tsv": "test_easy.tsv",
|
||||
"test_hard_queries_v3.tsv": "test_hard.tsv",
|
||||
}
|
||||
|
||||
|
||||
def download_text2sparql(infold: str):
|
||||
"""Downloads text2sparql train, test_easy, and test_hard data
|
||||
|
||||
Args:
|
||||
infold: save directory path
|
||||
"""
|
||||
os.makedirs(infold, exist_ok=True)
|
||||
|
||||
for prefix in prefix_map:
|
||||
url = base_url + prefix
|
||||
|
||||
logging.info(f"Downloading: {url}")
|
||||
if if_exist(infold, [prefix]):
|
||||
logging.info("** Download file already exists, skipping download")
|
||||
else:
|
||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with open(os.path.join(infold, prefix), "wb") as handle:
|
||||
handle.write(urlopen(req, timeout=20).read())
|
||||
|
||||
|
||||
def process_text2sparql(infold: str, outfold: str, do_lower_case: bool):
|
||||
""" Process and convert MeetKai's text2sparql datasets to NeMo's neural machine translation format.
|
||||
|
||||
Args:
|
||||
infold: directory path to raw text2sparql data containing
|
||||
train.tsv, test_easy.tsv, test_hard.tsv
|
||||
outfold: output directory path to save formatted data for NeuralMachineTranslationDataset
|
||||
the first line is header (sentence [tab] label)
|
||||
each line should be [sentence][tab][label]
|
||||
do_lower_case: if true, convert all sentences and labels to lower
|
||||
"""
|
||||
logging.info(f"Processing Text2Sparql dataset and storing at: {outfold}")
|
||||
|
||||
os.makedirs(outfold, exist_ok=True)
|
||||
|
||||
dataset_name = "Text2Sparql"
|
||||
for prefix in prefix_map:
|
||||
input_file = os.path.join(infold, prefix)
|
||||
output_file = os.path.join(outfold, prefix_map[prefix])
|
||||
|
||||
if if_exist(outfold, [prefix_map[prefix]]):
|
||||
logging.info(f"** {MODE_EXISTS_TMP.format(prefix_map[prefix], dataset_name, output_file)}")
|
||||
continue
|
||||
|
||||
if not if_exist(infold, [prefix]):
|
||||
logging.info(f"** {prefix} of {dataset_name}" f" is skipped as it was not found")
|
||||
continue
|
||||
|
||||
assert input_file != output_file, "input file cannot equal output file"
|
||||
with open(input_file, "r") as in_file:
|
||||
with open(output_file, "w") as out_file:
|
||||
reader = csv.reader(in_file, delimiter="\t")
|
||||
|
||||
# replace headers
|
||||
out_file.write("sentence\tlabel\n")
|
||||
next(reader)
|
||||
|
||||
for line in reader:
|
||||
sentence = line[0]
|
||||
label = line[1]
|
||||
if do_lower_case:
|
||||
sentence = sentence.lower()
|
||||
label = label.lower()
|
||||
out_file.write(f"{sentence}\t{label}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Parse the command-line arguments.
|
||||
parser = argparse.ArgumentParser(description="Process and convert datasets into NeMo's format")
|
||||
parser.add_argument(
|
||||
"--source_data_dir", required=True, type=str, help="Path to the folder containing the dataset files"
|
||||
)
|
||||
parser.add_argument("--target_data_dir", required=True, type=str, help="Path to save the processed dataset")
|
||||
parser.add_argument("--do_lower_case", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
source_dir = args.source_data_dir
|
||||
target_dir = args.target_data_dir
|
||||
do_lower_case = args.do_lower_case
|
||||
|
||||
download_text2sparql(infold=source_dir)
|
||||
process_text2sparql(infold=source_dir, outfold=target_dir, do_lower_case=do_lower_case)
|
||||
@@ -0,0 +1,72 @@
|
||||
# Copyright (c) 2020, MeetKai Inc. All rights reserved.
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script contains an example on how to evaluate a NeuralMachineTranslationModel.
|
||||
To load the example Text2Sparql dataset, please refer to ./data/import_datasets.py.
|
||||
To train a model, please refer to text2sparql.py.
|
||||
|
||||
|
||||
***Setting the configs***
|
||||
This script uses the `/examples/nlp/text2sparql/conf/text2sparql_config.yaml` config file by default.
|
||||
You may update the config file from the file directly or by using the command line arguments.
|
||||
Another other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'.
|
||||
|
||||
Please refer to text2sparql.py for detailed instructions on setting the configuration.
|
||||
|
||||
|
||||
***How to run the script?***
|
||||
- To reload and evaluate the model, run:
|
||||
|
||||
python evaluate_text2sparql.py \
|
||||
model.test_ds.filepath="$TGT_DATA_DIR"/test_easy.tsv \
|
||||
model.batch_size=16 \
|
||||
model.nemo_path=./NeMo_logs/bart.nemo \
|
||||
exp_manager.exp_dir=./NeMo_logs
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from nemo.collections.nlp.models.neural_machine_translation import NeuralMachineTranslationModel
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils import logging
|
||||
|
||||
|
||||
@hydra_runner(config_path="conf", config_name="text2sparql_config")
|
||||
def main(cfg: DictConfig) -> None:
|
||||
logging.info(f"Config:\n {OmegaConf.to_yaml(cfg)}")
|
||||
trainer = pl.Trainer(gpus=cfg.trainer.gpus)
|
||||
nmt_model = NeuralMachineTranslationModel.restore_from(restore_path=cfg.model.nemo_path)
|
||||
nmt_model.setup_test_data(cfg.model.test_ds)
|
||||
results = trainer.test(nmt_model)
|
||||
|
||||
with open(cfg.model.test_ds.filepath, "r") as f:
|
||||
lines = f.readlines()
|
||||
|
||||
lines[0] = lines[0].strip() + f"\tpredictions\n"
|
||||
for i, res in enumerate(results[0]["texts"]):
|
||||
lines[i + 1] = lines[i + 1].strip() + f"\t{res}\n"
|
||||
|
||||
savepath = os.path.join(cfg.exp_manager.exp_dir, os.path.basename(cfg.model.test_ds.filepath))
|
||||
with open(savepath, "w") as f:
|
||||
f.writelines(lines)
|
||||
logging.info(f"Predictions saved to {savepath}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright (c) 2020, MeetKai Inc. All rights reserved.
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
This script contains an example on how to train and save a NeuralMachineTranslationModel.
|
||||
NeuralMachineTranslationModel in NeMo supports sequence to sequence problems such as language translation
|
||||
and text summarization, provided the data follows the format specified below.
|
||||
|
||||
|
||||
***Data format***
|
||||
NeuralMachineTranslationModel requires the data to be stored in TAB separated files (.tsv) with two columns of
|
||||
sentence and label, where the first line is a header of format:
|
||||
sentence[TAB]label
|
||||
And each line is of the format:
|
||||
[SENTENCE][TAB][LABEL]
|
||||
|
||||
If your dataset is stored in another format, you need to convert it to this format to use a
|
||||
NeuralMachineTranslationModel.
|
||||
|
||||
|
||||
***Setting the configs***
|
||||
This script uses the `/examples/nlp/text2sparql/conf/text2sparql_config.yaml` config file by default.
|
||||
You may update the config file from the file directly or by using the command line arguments.
|
||||
Another other option is to set another config file via command line arguments by `--config-name=CONFIG_FILE_PATH'.
|
||||
|
||||
A NeuralMachineTranslationModel's config file declares multiple import sections. They are:
|
||||
- trainer: Arguments to be passed to PyTorch Lightning.
|
||||
- model: All arguments that relate to the Model - language_model, tokenizers, datasets, optimizer, generate.
|
||||
- exp_manager: Arguments to be passed to NeMo's experiment manager.
|
||||
- hydra: Arguments to be passed to Hydra.
|
||||
|
||||
If using text2sparql_config.yaml, you must first update the following fields in the config:
|
||||
- model.nemo_path: Model save path. Eg. [PATH]/bart.nemo
|
||||
- model.data_dir: Path to data directory. Alternatively, you can adjust the file paths directly:
|
||||
- model.train_ds.filepath
|
||||
- model.validation_ds.filepath
|
||||
- model.test_ds.filepath
|
||||
- exp_manager.exp_dir: Directory to log results from the experiment.
|
||||
|
||||
It is highly recommended to also adjust these parameters as necessary:
|
||||
- trainer.gpus: Set to 0 to use CPU. Otherwise the number denotes the number of GPUs.
|
||||
- trainer.max_epochs: Maximum number of epochs to train for.
|
||||
- model.batch_size: 8 is sufficient to train a decent Bart model for Text2Sparql.
|
||||
- model.max_seq_length: Maximum (tokenized) sequence length. 150 is sufficient for Text2Sparql.
|
||||
- model.language_model.pretrained_model_name: End2end pretrained model name from huggingface.
|
||||
- model.encoder_tokenizer.tokenizer_name: Pretrained tokenizer name from huggingface.
|
||||
- model.decoder_tokenizer.tokenizer_name: The same as above, as the tokenizer will handle encoding and decoding.
|
||||
- model.optim.lr: Learning rate.
|
||||
|
||||
You can also specify an encoder and decoder rather than using an end2end model like Bart by defining these parameters:
|
||||
- model.language_model.pretrained_encoder_model_name: Pretrained huggingface encoder model name.
|
||||
- model.encoder_tokenizer.tokenizer_name: Pretrained huggingface encoder tokenizer name.
|
||||
- model.language_model.pretrained_decoder_model_name: Pretrained huggingface decoder model name.
|
||||
- model.decoder_tokenizer.tokenizer_name: Pretrained huggingface decoder tokenizer name.
|
||||
- model.language_model.pretrained_model_name: Set this to null.
|
||||
|
||||
|
||||
***How to run the script?***
|
||||
- First, download the data to TGT_DATA_DIR (see ./data/import_datasets.py):
|
||||
|
||||
SRC_DATA_DIR=./data/text2sparql_src
|
||||
TGT_DATA_DIR=./data/text2sparql_tgt
|
||||
|
||||
python ./data/import_datasets.py \
|
||||
--source_data_dir $SRC_DATA_DIR \
|
||||
--target_data_dir $TGT_DATA_DIR
|
||||
|
||||
- And run the following to train and save the model:
|
||||
|
||||
python text2sparql.py \
|
||||
model.train_ds.filepath="$TGT_DATA_DIR"/train.tsv \
|
||||
model.validation_ds.filepath="$TGT_DATA_DIR"/test_easy.tsv \
|
||||
model.test_ds.filepath="$TGT_DATA_DIR"/test_hard.tsv \
|
||||
model.batch_size=16 \
|
||||
model.nemo_path=./NeMo_logs/bart.nemo \
|
||||
exp_manager.exp_dir=./NeMo_logs
|
||||
"""
|
||||
|
||||
import pytorch_lightning as pl
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from nemo.collections.nlp.models.neural_machine_translation import NeuralMachineTranslationModel
|
||||
from nemo.core.config import hydra_runner
|
||||
from nemo.utils import logging
|
||||
from nemo.utils.exp_manager import exp_manager
|
||||
|
||||
|
||||
@hydra_runner(config_path="conf", config_name="text2sparql_config")
|
||||
def main(cfg: DictConfig) -> None:
|
||||
logging.info(f"Config:\n {OmegaConf.to_yaml(cfg)}")
|
||||
trainer = pl.Trainer(**cfg.trainer)
|
||||
exp_manager(trainer, cfg.get("exp_manager", None))
|
||||
nmt_model = NeuralMachineTranslationModel(cfg.model, trainer=trainer)
|
||||
trainer.fit(nmt_model)
|
||||
if cfg.model.nemo_path:
|
||||
nmt_model.save_to(cfg.model.nemo_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user