Initial commit (move files)

moji/global_variables.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+""" Global variables.
+"""
+import tempfile
+from os.path import abspath, dirname
+
+# The ordering of these special tokens matter
+# blank tokens can be used for new purposes
+# Tokenizer should be updated if special token prefix is changed
+SPECIAL_PREFIX = 'CUSTOM_'
+SPECIAL_TOKENS = ['CUSTOM_MASK',
+                  'CUSTOM_UNKNOWN',
+                  'CUSTOM_AT',
+                  'CUSTOM_URL',
+                  'CUSTOM_NUMBER',
+                  'CUSTOM_BREAK']
+SPECIAL_TOKENS.extend(['{}BLANK_{}'.format(SPECIAL_PREFIX, i) for i in range(6, 10)])
+
+ROOT_PATH = dirname(dirname(abspath(__file__)))
+VOCAB_PATH = '{}/moji/model/vocabulary.json'.format(ROOT_PATH)
+PRETRAINED_PATH = '{}/moji/model/pytorch_model.bin'.format(ROOT_PATH)
+
+WEIGHTS_DIR = tempfile.mkdtemp()
+
+NB_TOKENS = 50000
+NB_EMOJI_CLASSES = 64
+FINETUNING_METHODS = ['last', 'full', 'new', 'chain-thaw']
+FINETUNING_METRICS = ['acc', 'weighted']