Introduce code from Rust monorepo. This is a mess

tts/NeMo/tools/text_processing_deployment/Dockerfile

@@ -0,0 +1,38 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2015 and onwards Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for C++ (inverse) text normalization backend Sparrowhawk https://github.com/google/sparrowhawk
+
+# set base image (host OS)
+FROM conda/miniconda3
+
+# set the working directory in the container
+WORKDIR /workspace
+
+# install dependencies
+RUN conda update -n base -c defaults conda -y
+RUN conda install conda-build -y
+RUN apt-get update &&     apt-get install -y --reinstall build-essential &&     apt-get upgrade -y &&     apt-get install -y git &&     apt-get install make
+RUN git clone https://github.com/google/re2
+RUN cd re2 && make && make install
+RUN apt-get install build-essential -y && apt-get install wget -y
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
+RUN tar xzvf protobuf-2.5.0.tar.gz
+RUN cd protobuf-2.5.0 && ./configure && make && make install && ldconfig
+RUN conda install -c conda-forge thrax=1.3.4 -y
+RUN git clone https://github.com/yzhang123/sparrowhawk.git
+RUN cd sparrowhawk &&  git checkout test &&   apt-get install -y autoconf &&     bash autoreconf && ./configure && make && make install && ldconfig
+RUN git clone https://github.com/kward/shunit2.git
+RUN echo "DONE"

tts/NeMo/tools/text_processing_deployment/README.rst

@@ -0,0 +1,10 @@
+**Text Processing Deployment**
+=========================================
+
+Introduction
+------------
+
+This folder provides scripts to deploy WFST-based grammars in `NeMo Text Processing <https://github.com/NVIDIA/NeMo/blob/main/nemo_text_processing>`_ for
+for production.
+
+See `documentation <https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/main/>`_ for details.

tts/NeMo/tools/text_processing_deployment/docker/build.sh

@@ -0,0 +1,18 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#!/bin/bash
+
+docker build . --rm -t sparrowhawk

tts/NeMo/tools/text_processing_deployment/docker/launch.sh

@@ -0,0 +1,44 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+MODE=${1:-"export"}
+SCRIPT_DIR=$(cd $(dirname $0); pwd)
+: ${CLASSIFY_DIR:="$SCRIPT_DIR/../classify"}
+: ${VERBALIZE_DIR:="$SCRIPT_DIR/../verbalize"}
+: ${CMD:=${2:-"/bin/bash"}}
+
+MOUNTS=""
+MOUNTS+=" -v $CLASSIFY_DIR:/workspace/sparrowhawk/documentation/grammars/en_toy/classify"
+MOUNTS+=" -v $VERBALIZE_DIR:/workspace/sparrowhawk/documentation/grammars/en_toy/verbalize"
+
+WORK_DIR="/workspace/sparrowhawk/documentation/grammars"
+if [[ $MODE == "test_tn_grammars" ]]; then
+  CMD="bash test_sparrowhawk_normalization.sh"
+  WORK_DIR="/workspace/tests"
+elif [[ $MODE == "test_itn_grammars" ]]; then
+  CMD="bash test_sparrowhawk_inverse_text_normalization.sh"
+  WORK_DIR="/workspace/tests"
+fi
+
+echo $MOUNTS
+docker run -it --rm \
+  --shm-size=4g \
+  --ulimit memlock=-1 \
+  --ulimit stack=67108864 \
+  $MOUNTS \
+  -v $SCRIPT_DIR/../../../tests/nemo_text_processing/:/workspace/tests \
+  -w $WORK_DIR \
+  sparrowhawk:latest $CMD

tts/NeMo/tools/text_processing_deployment/export_grammars.sh

@@ -0,0 +1,59 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2015 and onwards Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+# This script compiles and exports WFST-grammars from nemo_text_processing, builds C++ production backend Sparrowhawk (https://github.com/google/sparrowhawk) in docker, 
+# plugs grammars into Sparrowhawk and returns prompt inside docker.
+# For inverse text normalization run:
+#       bash export_grammars.sh --GRAMMARS=itn_grammars
+#       echo "two dollars fifty" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto
+# For text normalization run:
+#       bash export_grammars.sh --GRAMMARS=tn_grammars
+#       echo "\$2.5" | ../../src/bin/normalizer_main --config=sparrowhawk_configuration.ascii_proto
+#
+# To test TN grammars, run:
+#       bash export_grammars.sh --GRAMMARS=tn_grammars --MODE=test
+#
+# To test ITN grammars, run:
+#       bash export_grammars.sh --GRAMMARS=itn_grammars --MODE=test
+
+GRAMMARS="itn_grammars" # tn_grammars
+INPUT_CASE="cased" # lower_cased, only for tn_grammars
+MODE=""
+
+for ARG in "$@"
+do
+    key=$(echo $ARG | cut -f1 -d=)
+    value=$(echo $ARG | cut -f2 -d=)
+
+    if [[ $key == *"--"* ]]; then
+        v="${key/--/}"
+        declare $v="${value}"
+    fi
+done
+
+echo "GRAMMARS = $GRAMMARS"
+echo "MODE = $MODE"
+
+python pynini_export.py --output_dir=. --grammars=${GRAMMARS} --input_case=${INPUT_CASE} || exit 1
+find . -name "Makefile" -type f -delete
+bash docker/build.sh
+
+if [[ $MODE == "test" ]]; then
+  MODE=${MODE}_${GRAMMARS}
+fi
+
+bash docker/launch.sh $MODE

tts/NeMo/tools/text_processing_deployment/pynini_export.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+# Copyright 2015 and onwards Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import time
+from argparse import ArgumentParser
+from typing import Dict
+
+from nemo_text_processing.inverse_text_normalization.taggers.tokenize_and_classify import ClassifyFst as ITNClassifyFst
+from nemo_text_processing.inverse_text_normalization.verbalizers.verbalize import VerbalizeFst as ITNVerbalizeFst
+from nemo_text_processing.text_normalization.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst
+from nemo_text_processing.text_normalization.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst
+
+from nemo.utils import logging
+
+try:
+    import pynini
+    from pynini.export import export
+
+    PYNINI_AVAILABLE = True
+except (ModuleNotFoundError, ImportError):
+
+    logging.warning(
+        "`pynini` is not installed ! \n"
+        "Please run the `nemo_text_processing/setup.sh` script"
+        "prior to usage of this toolkit."
+    )
+
+    PYNINI_AVAILABLE = False
+
+
+# This script exports compiled grammars inside nemo_text_processing into OpenFst finite state archive files tokenize_and_classify.far and verbalize.far for production purposes
+
+
+def _generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]):
+    """
+    Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. 
+
+    Args:
+        file_name: exported file name
+        graph: Pynini WFST graph to be exported
+        rule_name: rule name for graph in created FAR file
+
+    """
+    exporter = export.Exporter(file_name)
+    for rule, graph in graphs.items():
+        exporter[rule] = graph.optimize()
+    exporter.close()
+    print(f'Created {file_name}')
+
+
+def itn_grammars(**kwargs):
+    d = {}
+    d['classify'] = {'TOKENIZE_AND_CLASSIFY': ITNClassifyFst().fst}
+    d['verbalize'] = {'ALL': ITNVerbalizeFst().fst, 'REDUP': pynini.accep("REDUP")}
+    return d
+
+
+def tn_grammars(**kwargs):
+    d = {}
+    d['classify'] = {'TOKENIZE_AND_CLASSIFY': TNClassifyFst(input_case=kwargs["input_case"], deterministic=True).fst}
+    d['verbalize'] = {'ALL': TNVerbalizeFst(deterministic=True).fst, 'REDUP': pynini.accep("REDUP")}
+    return d
+
+
+def export_grammars(output_dir, grammars):
+    """
+    Exports tokenizer_and_classify and verbalize Fsts as OpenFst finite state archive (FAR) files. 
+
+    Args:
+        output_dir: directory to export FAR files to. Subdirectories will be created for tagger and verbalizer respectively.
+    """
+
+    for category, graphs in grammars.items():
+        out_dir = os.path.join(output_dir, category)
+        if not os.path.exists(out_dir):
+            os.makedirs(out_dir)
+            time.sleep(1)
+        if category == "classify":
+            category = "tokenize_and_classify"
+        _generator_main(f"{out_dir}/{category}.far", graphs)
+
+
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument("--output_dir", help="output directory for grammars", required=True, type=str)
+    parser.add_argument(
+        "--grammars", help="grammars to be exported", choices=["tn_grammars", "itn_grammars"], type=str, required=True
+    )
+    parser.add_argument(
+        "--input_case", help="input capitalization", choices=["lower_cased", "cased"], default="cased", type=str
+    )
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    export_grammars(output_dir=args.output_dir, grammars=locals()[args.grammars](input_case=args.input_case))