mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-17 00:23:14 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
+24
@@ -0,0 +1,24 @@
|
||||
nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130
|
||||
two hundred and fifty four~254
|
||||
one hundred forty seven thousand four hundred fifty one~147451
|
||||
one million one hundred fifty six thousand one hundred seventy three~1156173
|
||||
one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961
|
||||
ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005
|
||||
seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949
|
||||
ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010
|
||||
minus twenty five thousand thirty seven~-25037
|
||||
one quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104
|
||||
minus sixty~-60
|
||||
forty six thousand six hundred sixty four~46664
|
||||
sixty~60
|
||||
zero~zero
|
||||
two million three~2000003
|
||||
one thousand thirteen~1013
|
||||
one thousand one~1001
|
||||
one thousand one hundred~1100
|
||||
one thousand twenty six~1026
|
||||
one thousand one hundred twenty six~1126
|
||||
eighteen million four hundred fifty thousand nine hundred ninety~18450990
|
||||
eighteen million nine hundred forty thousand seven hundred twenty two~18940722
|
||||
eighteen million six hundred ninety thousand nine hundred sixteen~18690916
|
||||
eighteen thousand eight hundred eighty~18880
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
july twenty fifth two thousand twelve~july 25 2012
|
||||
nineteen eighties~1980s
|
||||
two thousand and twenty~2020
|
||||
two thousand and nine~2009
|
||||
the twenty fifth of july twenty twelve~25 july 2012
|
||||
the twenty fifth of july two thousand twelve~25 july 2012
|
||||
the twenty second of july twenty twelve~22 july 2012
|
||||
the fifteenth of january~15 january
|
||||
the seventeenth of may twenty ten~17 may 2010
|
||||
january first~january 1
|
||||
july twenty second two thousand eight~july 22 2008
|
||||
june thirty~june 30
|
||||
july twenty fifth twenty twelve~july 25 2012
|
||||
nineteen seventeen~1917
|
||||
twenty twelve~2012
|
||||
march sixteen sixty five~march 1665
|
||||
sixteen sixty five~1665
|
||||
july two thousand twelve~july 2012
|
||||
october nineteen oh five~october 1905
|
||||
july fifteen o six~july 1506
|
||||
the twenty fifth of july twenty twelve~25 july 2012
|
||||
july twenty fifth twenty twelve~july 25 2012
|
||||
july twenty fifth two thousand twelve~july 25 2012
|
||||
july one thousand eight hundred seventy six~july 1876
|
||||
february twenty fifth twenty sixteen~february 25 2016
|
||||
november twenty fourth twenty fourteen~november 24 2014
|
||||
nineteen ninety four~1994
|
||||
two thousand three~2003
|
||||
one thousand eight~1008
|
||||
nineteen seventy six~1976
|
||||
june twentieth twenty fourteen~june 20 2014
|
||||
nineteen seventy three~1973
|
||||
nineteen seventy five~1975
|
||||
eleven fifty five~1155
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
five point two million~5.2 million
|
||||
one hundred sixty four point five eight thousand~164.58 thousand
|
||||
four hundred million~400 million
|
||||
fifty billion~50 billion
|
||||
four hundred five billion~405 billion
|
||||
four point eight five billion~4.85 billion
|
||||
one hundred billion~100 billion
|
||||
one hundred ten billion~110 billion
|
||||
one hundred thirty two billion~132 billion
|
||||
one point eight four billion~1.84 billion
|
||||
one point eight one billion~1.81 billion
|
||||
one point five nine billion~1.59 billion
|
||||
one point four five three billion~1.453 billion
|
||||
one point seven two billion~1.72 billion
|
||||
one point two five billion~1.25 billion
|
||||
thirteen billion~13 billion
|
||||
thirty billion~30 billion
|
||||
two thousand eight hundred five point eight seven three billion~2805.873 billion
|
||||
seventy trillion~70 trillion
|
||||
thirteen million~13 million
|
||||
eighteen billion~18 billion
|
||||
four hundred fifty million~450 million
|
||||
one hundred thirty million~130 million
|
||||
ten million~10 million
|
||||
four hundred million~400 million
|
||||
five million~5 million
|
||||
five hundred million~500 million
|
||||
twelve million~12 million
|
||||
thirteen million~13 million
|
||||
four million~4 million
|
||||
forty five million~45 million
|
||||
fifteen million~15 million
|
||||
fifteen trillion~15 trillion
|
||||
fifteen billion~15 billion
|
||||
two million~2 million
|
||||
eight million~8 million
|
||||
point one two o five~.1205
|
||||
minus sixty point two four zero zero~-60.2400
|
||||
zero point two six~0.26
|
||||
point zero two~.02
|
||||
sixty point two~60.2
|
||||
eighteen~18
|
||||
eighteen point eight five~18.85
|
||||
eighteen point five o~18.50
|
||||
eighteen point five six~18.56
|
||||
eighteen point nine~18.9
|
||||
eighteen point o five~18.05
|
||||
eighteen point one two~18.12
|
||||
eighteen point o one~18.01
|
||||
eighteen point o o o~18.000
|
||||
eighteen point six~18.6
|
||||
eighteen point three o o~18.300
|
||||
eighteen point three six~18.36
|
||||
eighteen point two five~18.25
|
||||
eighteen point two two~18.22
|
||||
eight hundred eighteen point three o three~818.303
|
||||
eight hundred eight point eight~808.8
|
||||
eight hundred eight point zero~808.0
|
||||
eight hundred eighty eight point one~888.1
|
||||
eight hundred eighty four point three~884.3
|
||||
eight hundred eighty two point eight~882.8
|
||||
eight hundred eighty two point zero~882.0
|
||||
eight hundred forty five point nine four~845.94
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
a dot b c at g mail dot com~a.bc@gmail.com
|
||||
c d f at a b c dot e d u~cdf@abc.edu
|
||||
a b c at g mail dot a b c~abc@gmail.abc
|
||||
a b c at a b c dot com~abc@abc.com
|
||||
a s d f one two three at a b c dot com~asdf123@abc.com
|
||||
a one b two at a b c dot com~a1b2@abc.com
|
||||
a b three dot s d d dot three at g mail dot com~ab3.sdd.3@gmail.com
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
two hundred meters~200 m
|
||||
fifty six point three per square kilometer~56.3 /km²
|
||||
two hundred kilometers per hour~200 km/h
|
||||
twenty eight kilograms force per square centimeter~28 kgf/cm²
|
||||
forty two thousand two hundred fifty nine per square meter~42259 /m²
|
||||
minus two thousand twelve kilo liters~-2012 kl
|
||||
minus sixty six kilograms~-66 kg
|
||||
two kilo watt hours~2 kwh
|
||||
one point o o o o two eight cubic deci meters~1.000028 dm³
|
||||
seven point five peta bytes~7.5 pb
|
||||
three hours~3 h
|
||||
one milli volt~1 mv
|
||||
two cubic meters~2 m³
|
||||
ninety grams~90 g
|
||||
one hundred twenty four point three lumens~124.3 lm
|
||||
four hundred forty milliliters~440 ml
|
||||
thirty one thousand four hundred eighty square feet~31480 sq ft
|
||||
one thousand six hundred hours~1600 h
|
||||
thirty one thousand four hundred eighty square feet~31480 sq ft
|
||||
two square miles~2 sq mi
|
||||
zero point one nine square miles~0.19 sq mi
|
||||
one thousand five hundred thirty one c c~1531 cc
|
||||
three hundred micrometers~300 μm
|
||||
sixty five thousand square kilometers~65000 km²
|
||||
two miles per hour~2 mph
|
||||
two hundred forty five miles per hour~245 mph
|
||||
one hundred fifty c c~150 cc
|
||||
sixty point two four zero zero kilograms~60.2400 kg
|
||||
zero feet~0 ft
|
||||
zero foot~zero foot
|
||||
two feet~2 ft
|
||||
twenty foot~20 foot
|
||||
point two meters~.2 m
|
||||
two square meters~2 m²
|
||||
eighteen feet~18 ft
|
||||
eighteen mega siemens~18 ms
|
||||
eighteen ounces~18 oz
|
||||
eighteen point five kilometers~18.5 km
|
||||
eighteen point five two square kilometers~18.52 km²
|
||||
eighteen point nine one square kilometers~18.91 km²
|
||||
eighteen point one four percent~18.14 %
|
||||
eighteen point one six percent~18.16 %
|
||||
eighteen point one square kilometers~18.1 km²
|
||||
eighteen point six percent~18.6 %
|
||||
eighteen point two two kilometers~18.22 km
|
||||
eighteen point zero kilometers~18.0 km
|
||||
eighteen point zero percent~18.0 %
|
||||
eighteen square kilometers~18 km²
|
||||
eighteen thousand eight hundred giga watt hours~18800 gwh
|
||||
eighteen thousand seven hundred hectares~18700 ha
|
||||
eight hectares~8 ha
|
||||
eight hundred eighty five astronomical units~885 au
|
||||
eight hundred eighty hectares~880 ha
|
||||
eight hundred eighty kilobytes~880 kb
|
||||
eight hundred eighty kilometers~880 km
|
||||
eight hundred eighty nine feet~889 ft
|
||||
eight hundred eighty six kilometers~886 km
|
||||
eight hundred eighty two megawatts~882 mw
|
||||
eight hundred feet~800 ft
|
||||
eight hundred fifty five square kilometers~855 km²
|
||||
eight hundred fifty megahertz~850 mhz
|
||||
eight hundred fifty meters~850 m
|
||||
eight hundred fifty nanometers~850 nm
|
||||
eight hundred fifty one meters~851 m
|
||||
eight hundred fifty seven square kilometers~857 km²
|
||||
eight hundred fifty three meters~853 m
|
||||
eight hundred fifty three point six meters~853.6 m
|
||||
eight hundred five point four six square kilometers~805.46 km²
|
||||
eight hundred forty two point nine meters~842.9 m
|
||||
eight hundred forty two square kilometers~842 km²
|
||||
eight hundred gigabytes~800 gb
|
||||
eight hundred horsepower~800 hp
|
||||
eight hundred kilograms~800 kg
|
||||
eight hundred kilo watt hours~800 kwh
|
||||
eight hundred kilowatts~800 kw
|
||||
eight hundred megahertz~800 mhz
|
||||
eight hundred ninety four c c~894 cc
|
||||
eight hundred ninety kilowatts~890 kw
|
||||
eight hundred ninety millimeters~890 mm
|
||||
eight hundred ninety two square kilometers~892 km²
|
||||
eight hundred seventy horsepower~870 hp
|
||||
eight hundred seventy meters~870 m
|
||||
eight hundred sixty kilograms~860 kg
|
||||
eight hundred sixty kilometers~860 km
|
||||
eight hundred sixty miles~860 mi
|
||||
eight hundred sixty six feet~866 ft
|
||||
eight hundred ten hectares~810 ha
|
||||
eight hundred ten kilohertz~810 khz
|
||||
eight hundred thirty eight point two millimeters~838.2 mm
|
||||
eight hundred thirty five kilometers~835 km
|
||||
eight hundred thirty kilohertz~830 khz
|
||||
eight hundred thirty megawatts~830 mw
|
||||
eight hundred thirty nine kilometers~839 km
|
||||
eight hundred thirty six meters~836 m
|
||||
eight hundred twenty feet~820 ft
|
||||
eight hundred twenty kilometers~820 km
|
||||
eight hundred twenty meters~820 m
|
||||
eight hundred twenty one point zero feet~821.0 ft
|
||||
eight hundred two point eight nine kilometers~802.89 km
|
||||
eight hundred volts~800 v
|
||||
eight kilobits~8 kb
|
||||
eight kilograms~8 kg
|
||||
eight million two hundred thousand feet~8200000 ft
|
||||
eight point eight kilometers~8.8 km
|
||||
eight point eight meters~8.8 m
|
||||
eight point eight miles~8.8 mi
|
||||
eight point five centimeters~8.5 cm
|
||||
eight point five five percent~8.55 %
|
||||
eight point five megawatts~8.5 mw
|
||||
eight point five meters~8.5 m
|
||||
eight point five two percent~8.52 %
|
||||
eight point four four percent~8.44 %
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
two dollars~$2
|
||||
one cent~$0.01
|
||||
four united states dollars and sixty nine cents~$4.69
|
||||
seventy five dollars sixty three~$75.63
|
||||
twenty nine dollars fifty cents~$29.50
|
||||
eleven dollars and fifty one cents~$11.51
|
||||
nine hundred ninety three dollars and ninety two cents~$993.92
|
||||
four hundred sixty billion won~₩460 billion
|
||||
thirty billion yen~¥30 billion
|
||||
two point five billion dollars~$2.5 billion
|
||||
forty five billion dollars~$45 billion
|
||||
fifty million dollars~$50 million
|
||||
fifty billion dollars~$50 billion
|
||||
zero point two million dollars~$0.2 million
|
||||
fifteen point two billion dollars~$15.2 billion
|
||||
one point six nine billion yuan~1.69 billion yuan
|
||||
one point four three six billion yuan~1.436 billion yuan
|
||||
four million yuan~4 million yuan
|
||||
one dollar~$1
|
||||
fifteen thousand dollars~$15000
|
||||
one dollars~one dollars
|
||||
twenty dollar~20 dollar
|
||||
twenty point five o six dollars~$20.506
|
||||
point five o six dollars~$.506
|
||||
eighteen dollars~$18
|
||||
eighteen million nine hundred twenty five thousand dollars~$18925000
|
||||
eighteen thousand eight hundred fifty four dollars~$18854
|
||||
eighteen thousand eight hundred one dollars~$18801
|
||||
eighteen thousand eight hundred seventy five dollars~$18875
|
||||
eighteen thousand eighty one dollars~$18081
|
||||
eighteen thousand fifty two dollars~$18052
|
||||
eighteen thousand five hundred forty two dollars~$18542
|
||||
eighteen thousand five hundred nineteen dollars~$18519
|
||||
eighteen thousand five hundred seventy dollars~$18570
|
||||
eighteen thousand five hundred seventy eight dollars~$18578
|
||||
eighteen thousand five hundred sixteen dollars~$18516
|
||||
eighteen thousand four hundred eighty two dollars~$18482
|
||||
eighteen thousand four hundred seventy eight dollars~$18478
|
||||
eighteen thousand four hundred sixty eight dollars~$18468
|
||||
eighteen thousand nine hundred three dollars~$18903
|
||||
eighteen thousand nine hundred twenty nine dollars~$18929
|
||||
eighteen thousand ninety five dollars~$18095
|
||||
eighteen thousand one hundred seventeen dollars~$18117
|
||||
eighteen thousand one hundred twenty eight dollars~$18128
|
||||
eighteen thousand one hundred twenty five dollars~$18125
|
||||
eighteen thousand one hundred twenty four dollars~$18124
|
||||
eighteen thousand one hundred twenty nine dollars~$18129
|
||||
eighteen thousand pounds~£18000
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
one hundredth~100th
|
||||
twenty five thousand one hundred eleventh~25111th
|
||||
second~2nd
|
||||
zeroth~0th
|
||||
first~1st
|
||||
second~2nd
|
||||
third~3rd
|
||||
fourth~4th
|
||||
eleventh~11th
|
||||
twelfth~12th
|
||||
thirteenth~13th
|
||||
twenty first~21st
|
||||
twenty third~23rd
|
||||
one hundred eleventh~111th
|
||||
one thousandth~1000th
|
||||
one hundred twenty first~121st
|
||||
second~2nd
|
||||
tenth~10th
|
||||
sixth~6th
|
||||
third~3rd
|
||||
nineteenth~19th
|
||||
third~3rd
|
||||
twelfth~12th
|
||||
forty eighth~48th
|
||||
seventy first~71st
|
||||
third~3rd
|
||||
forty second~42nd
|
||||
seventeenth~17th
|
||||
twentieth~20th
|
||||
twenty first~21st
|
||||
seventh~7th
|
||||
second~2nd
|
||||
fifth~5th
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
one two three one two three five six seven eight~123-123-5678
|
||||
zero two three one two three five six seven eight~023-123-5678
|
||||
o two three one two three five six seven eight~023-123-5678
|
||||
oh two three one two three five six seven eight~023-123-5678
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
eight oclock g m t~08:00 gmt
|
||||
seven a m e s t~07:00 a.m. est
|
||||
two p m~02:00 p.m.
|
||||
two thirty~02:30
|
||||
three o'clock~03:00
|
||||
quarter past one~01:15
|
||||
half past three~03:30
|
||||
eight fifty one~08:51
|
||||
eight fifty two~08:52
|
||||
eight forty~08:40
|
||||
eight nineteen~08:19
|
||||
eight o six~08:06
|
||||
eight thirty eight~08:38
|
||||
eight thirty two~08:32
|
||||
eight twenty nine~08:29
|
||||
eleven fifty five p m~11:55 p.m.
|
||||
eleven fifty three p m~11:53 p.m.
|
||||
eleven forty a m~11:40 a.m.
|
||||
eleven forty five a m~11:45 a.m.
|
||||
eleven forty p m~11:40 p.m.
|
||||
eleven forty six a m~11:46 a.m.
|
||||
eleven o six p m~11:06 p.m.
|
||||
eleven thirteen a m~11:13 a.m.
|
||||
half past twelve~12:30
|
||||
quarter past one~01:15
|
||||
quarter to one~12:45
|
||||
quarter to twelve~11:45
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
doctor dao~dr. dao
|
||||
miss smith~ms. smith
|
||||
misses smith~mrs. smith
|
||||
mister dao~mr. dao
|
||||
saint george~st. george
|
||||
i like for example ice cream~i like e.g. ice cream
|
||||
s and p five hundred~s&p 500
|
||||
seven eleven stores~7-eleven stores
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
~
|
||||
e s three~es3
|
||||
yahoo!~yahoo!
|
||||
twenty!~20 !
|
||||
x ~x
|
||||
—~—
|
||||
aaa~aaa
|
||||
aabach~aabach
|
||||
aabenraa~aabenraa
|
||||
aabye~aabye
|
||||
aaccessed~aaccessed
|
||||
aach~aach
|
||||
aachen's~aachen's
|
||||
aadri~aadri
|
||||
aafia~aafia
|
||||
aagaard~aagaard
|
||||
aagadu~aagadu
|
||||
aagard~aagard
|
||||
aagathadi~aagathadi
|
||||
aaghart's~aaghart's
|
||||
aagnes~aagnes
|
||||
aagomoni~aagomoni
|
||||
aagon~aagon
|
||||
aagoo~aagoo
|
||||
aagot~aagot
|
||||
aahar~aahar
|
||||
aahh~aahh
|
||||
aahperd~aahperd
|
||||
aaibinterstate~aaibinterstate
|
||||
aajab~aajab
|
||||
aakasa~aakasa
|
||||
aakervik~aakervik
|
||||
aakirkeby~aakirkeby
|
||||
aalam~aalam
|
||||
aalbaek~aalbaek
|
||||
aaldiu~aaldiu
|
||||
aalem~aalem
|
||||
a'ali~a'ali
|
||||
aalilaassamthey~aalilaassamthey
|
||||
aalin~aalin
|
||||
aaliyan~aaliyan
|
||||
aaliyan's~aaliyan's
|
||||
aamadu~aamadu
|
||||
aamara~aamara
|
||||
aambala~aambala
|
||||
aamera~aamera
|
||||
aamer's~aamer's
|
||||
aamina~aamina
|
||||
aaminah~aaminah
|
||||
aamjiwnaang~aamjiwnaang
|
||||
@@ -0,0 +1,7 @@
|
||||
1~one
|
||||
1~one
|
||||
1, ~one ,
|
||||
1!!!!~one ! ! ! !
|
||||
(1)Hello~(1)Hello
|
||||
!1~! one
|
||||
1!hello~1!hello
|
||||
@@ -0,0 +1,18 @@
|
||||
1~one
|
||||
2~two
|
||||
-2~minus two
|
||||
3~three
|
||||
123~one hundred twenty three
|
||||
13,000~thirteen thousand
|
||||
13,000~thirteen thousand
|
||||
123,123,000~one hundred twenty three million one hundred twenty three thousand
|
||||
123,000,012~one hundred twenty three million twelve
|
||||
t-0t25d12-f~t oh t two five d one two f
|
||||
133-ABC~one hundred thirty three ABC
|
||||
B2A23C~B two A two three C
|
||||
25d08A~two five d oh eight A
|
||||
C24~C two four
|
||||
W2s~W two s
|
||||
1-4-a-b-1-5~one four a b one five
|
||||
b-c-1-5-b-s-b~b c one five b s b
|
||||
1/f-4s~one f four s
|
||||
@@ -0,0 +1,32 @@
|
||||
july 25 2012~july twenty fifth twenty twelve
|
||||
jul 25 2012~july twenty fifth twenty twelve
|
||||
1980s~nineteen eighties
|
||||
1980 s~nineteen eighties
|
||||
25 july 2012~the twenty fifth of july twenty twelve
|
||||
25 jul 2012~the twenty fifth of july twenty twelve
|
||||
22 july 2012~the twenty second of july twenty twelve
|
||||
15 january~the fifteenth of january
|
||||
17 may 2010~the seventeenth of may twenty ten
|
||||
january 1~january first
|
||||
june 30~june thirtieth
|
||||
2012~twenty twelve
|
||||
july 1506~july fifteen o six
|
||||
july 1876~july eighteen seventy six
|
||||
1994~nineteen ninety four
|
||||
1976~nineteen seventy six
|
||||
june 20 2014~june twentieth twenty fourteen
|
||||
1973~nineteen seventy three
|
||||
1975~nineteen seventy five
|
||||
1155~eleven fifty five
|
||||
2006-08-05~august fifth two thousand six
|
||||
august 23, 2002~august twenty third two thousand two
|
||||
20 january~the twentieth of january
|
||||
2016-11-03~november third twenty sixteen
|
||||
2016/07/03~july third twenty sixteen
|
||||
1980s~nineteen eighties
|
||||
01.10.2010~january tenth twenty ten
|
||||
10/06/2005~october sixth two thousand five
|
||||
1910s~nineteen tens
|
||||
25 sept.~the twenty fifth of september
|
||||
1000~one thousand
|
||||
SEPT. 15TH~september fifteenth
|
||||
@@ -0,0 +1,8 @@
|
||||
.1665~point one six six five
|
||||
2.0~two point zero
|
||||
2.00~two point o o
|
||||
2.050~two point o five o
|
||||
100 million~one hundred million
|
||||
0.1 billion~zero point one billion
|
||||
.1 trillion~point one trillion
|
||||
-0.1~minus zero point one
|
||||
@@ -0,0 +1,10 @@
|
||||
a.bc@gmail.com~a dot b c at g mail dot com
|
||||
cdf@abc.edu~c d f at a b c dot e d u
|
||||
abc@gmail.abc~a b c at g mail dot a b c
|
||||
abc@abc.com~a b c at a b c dot com
|
||||
asdf123@abc.com~a s d f one two three at a b c dot com
|
||||
a1b2@abc.com~a one b two at a b c dot com
|
||||
ab3.sdd.3@gmail.com~a b three dot s d d dot three at g mail dot com
|
||||
email is abc1@gmail.com~email is a b c one at g mail dot com
|
||||
test .b@gmail.com~test . b at g mail dot com
|
||||
email is a-1.b3_c&-d@gma4i-l.com test~email is a dash one dot b three underscore c ampersand dash d at g m a four i dash l dot com test
|
||||
@@ -0,0 +1,11 @@
|
||||
1/2007~one two thousand seventh
|
||||
12639/12640~twelve thousand six hundred thirty nine twelve thousand six hundred fortieths
|
||||
2/4~two quarters
|
||||
1/4~one quarter
|
||||
31/32~thirty one thirty seconds
|
||||
22/3~twenty two thirds
|
||||
1/3~one third
|
||||
142/1~one hundred forty two over one
|
||||
1/2~one half
|
||||
2 1/2~two and a half
|
||||
1795 / 1805~one thousand seven hundred ninety five one thousand eight hundred fifths
|
||||
@@ -0,0 +1,15 @@
|
||||
12kg~twelve kilograms
|
||||
12/kg~twelve per kilogram
|
||||
12kg/kg~twelve kilograms per kilogram
|
||||
1 mbps~one megabit per second
|
||||
3 mbps~three megabits per second
|
||||
3 cc/s~three c c per second
|
||||
100 million kg~one hundred million kilograms
|
||||
a 4-kilogram bag~a four kilogram bag
|
||||
55-millimeters long~fifty five millimeters long
|
||||
a 123.2-millimeters long~a one hundred twenty three point two millimeters long
|
||||
covid-19.5~covid nineteen point five
|
||||
covid-19~covid nineteen
|
||||
a 4-kilogram bag~a four kilogram bag
|
||||
7.2-millimeter bullet~seven point two millimeter bullet
|
||||
4 1/2 lbs~four and a half pounds
|
||||
@@ -0,0 +1,38 @@
|
||||
$2~two dollars
|
||||
₩460 billion~four hundred sixty billion won
|
||||
¥30 billion~thirty billion yen
|
||||
$45 billion~forty five billion dollars
|
||||
$0.2 million~zero point two million dollars
|
||||
$15.2 billion~fifteen point two billion dollars
|
||||
1.436 billion yuan~one point four three six billion yuan
|
||||
4 million yuan~four million yuan
|
||||
$15000~fifteen thousand dollars
|
||||
$1~one dollar
|
||||
$20~twenty dollars
|
||||
$20.506~twenty point five o six dollars
|
||||
$.506~point five o six dollars
|
||||
$18~eighteen dollars
|
||||
$18925000~eighteen million nine hundred twenty five thousand dollars
|
||||
$18,925,000~eighteen million nine hundred twenty five thousand dollars
|
||||
$18854~eighteen thousand eight hundred fifty four dollars
|
||||
$18801~eighteen thousand eight hundred one dollars
|
||||
$18875~eighteen thousand eight hundred seventy five dollars
|
||||
$18081~eighteen thousand eighty one dollars
|
||||
$18052~eighteen thousand fifty two dollars
|
||||
$18542~eighteen thousand five hundred forty two dollars
|
||||
$18519~eighteen thousand five hundred nineteen dollars
|
||||
$18570~eighteen thousand five hundred seventy dollars
|
||||
$18578~eighteen thousand five hundred seventy eight dollars
|
||||
$18516~eighteen thousand five hundred sixteen dollars
|
||||
$18482~eighteen thousand four hundred eighty two dollars
|
||||
$18478~eighteen thousand four hundred seventy eight dollars
|
||||
$18468~eighteen thousand four hundred sixty eight dollars
|
||||
$18903~eighteen thousand nine hundred three dollars
|
||||
$18929~eighteen thousand nine hundred twenty nine dollars
|
||||
$18095~eighteen thousand ninety five dollars
|
||||
$18117~eighteen thousand one hundred seventeen dollars
|
||||
$18128~eighteen thousand one hundred twenty eight dollars
|
||||
$18125~eighteen thousand one hundred twenty five dollars
|
||||
$18124~eighteen thousand one hundred twenty four dollars
|
||||
$18129~eighteen thousand one hundred twenty nine dollars
|
||||
£18000~eighteen thousand pounds
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
~123
|
||||
one twenty three
|
||||
one two three
|
||||
one hundred twenty three
|
||||
one hundred and twenty three
|
||||
~$123
|
||||
one twenty three dollars
|
||||
one two three dollars
|
||||
one hundred twenty three dollars
|
||||
one hundred and twenty three dollars
|
||||
~$123.2
|
||||
one twenty three dollars and two cents
|
||||
one two three dollars two cents
|
||||
one hundred twenty three dollars and two cents
|
||||
one hundred and twenty three point two dollars
|
||||
~1.24
|
||||
one point two four
|
||||
one two four
|
||||
one point twenty four
|
||||
one twenty four
|
||||
~$1.21
|
||||
one dollar and twenty one cents
|
||||
one dollar twenty one cents
|
||||
one point two one dollars
|
||||
one twenty one dollars
|
||||
~£1.00
|
||||
one pound
|
||||
one point o o pounds
|
||||
~t-0t25d12-f
|
||||
t zero t twenty five d one two f
|
||||
t zero t twenty five d twelve f
|
||||
t zero t two five d one two f
|
||||
t zero t two five d twelve f
|
||||
t oh t two five d one two f
|
||||
t oh t twenty five d one two f
|
||||
t oh t twenty five d twelve f
|
||||
t oh t two five d twelve f
|
||||
~133-A
|
||||
one hundred thirty three A
|
||||
~B2A23C
|
||||
B two A twenty three C
|
||||
~25d08A
|
||||
twenty five d zero eight A
|
||||
~C24
|
||||
C twenty four
|
||||
~It seemed to her that the jacket Oswald wore was darker than Commission Exhibit No. 162.
|
||||
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one sixty two.
|
||||
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred sixty two.
|
||||
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred and sixty two.
|
||||
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one six two.
|
||||
~"Father, let this cup pass." He prayed--was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord:-- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim."
|
||||
"Father, let this cup pass." He prayed -- was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord: -- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim."
|
||||
~1970-2010
|
||||
nineteen seventy to twenty ten
|
||||
one thousand nine seventy to two thousand ten
|
||||
one thousand nine hundred and seventy to two thousand ten
|
||||
one thousand nine hundred and seventy to twenty ten
|
||||
nineteen seventy twenty ten
|
||||
one thousand nine seventy two thousand ten
|
||||
one thousand nine hundred and seventy two thousand ten
|
||||
one thousand nine hundred and seventy twenty ten
|
||||
from nineteen seventy to twenty ten
|
||||
from one thousand nine seventy to two thousand ten
|
||||
from one thousand nine hundred and seventy to two thousand ten
|
||||
from one thousand nine hundred and seventy to twenty ten
|
||||
from nineteen seventy twenty ten
|
||||
from one thousand nine seventy two thousand ten
|
||||
from one thousand nine hundred and seventy two thousand ten
|
||||
from one thousand nine hundred and seventy twenty ten
|
||||
~W26s
|
||||
W twenty sixes
|
||||
W twenty six s
|
||||
~5-3-Ws
|
||||
five three W s
|
||||
five three Ws
|
||||
~401-ks
|
||||
four oh one kays
|
||||
four oh one k s
|
||||
four hundred one k s
|
||||
~The box was 25 x 7 m.
|
||||
The box was twenty five by seven meters.
|
||||
The box was two five by seven meters.
|
||||
The box was two five by seven m.
|
||||
The box was twenty five by seven m.
|
||||
~4567
|
||||
four thousand five hundred sixty seven
|
||||
four five six seven
|
||||
four thousand five sixty seven
|
||||
forty five sixty seven
|
||||
four thousand five hundred and sixty seven
|
||||
~This example number 15,000 can be a very long one, and can fail to produce valid normalization for a such an easy number like 10,125 or dollar value $5349.01, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, 452.
|
||||
This example number fifteen thousand can be a very long one, and can fail to produce valid normalization for a such an easy number like ten thousand one hundred twenty five or dollar value five thousand three hundred forty nine dollars and one cent, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, four five two.
|
||||
~$1.01
|
||||
one dollar and one cent
|
||||
one dollar one cent
|
||||
one dollar and zero one cents
|
||||
~$17.31
|
||||
seventeen dollars and thirty one cent
|
||||
seventeen dollars and thirty one cents
|
||||
seventeen point three one dollars
|
||||
~25.]
|
||||
two five.]
|
||||
twenty five.]
|
||||
~Francis I--test
|
||||
Francis the first -- test
|
||||
Francis one -- test
|
||||
Francis first -- test
|
||||
@@ -0,0 +1,6 @@
|
||||
0th~zeroth
|
||||
11th~eleventh
|
||||
23rd~twenty third
|
||||
1000th~one thousandth
|
||||
1,000th~one thousandth
|
||||
1000000th~one millionth
|
||||
@@ -0,0 +1,9 @@
|
||||
+1 123-123-5678~one, one two three, one two three, five six seven eight
|
||||
123-123-5678~one two three, one two three, five six seven eight
|
||||
+1-123-123-5678~one, one two three, one two three, five six seven eight
|
||||
+1 (123)-123-5678~one, one two three, one two three, five six seven eight
|
||||
+1(123) 123-5678~one, one two three, one two three, five six seven eight
|
||||
(123)-123-5678~one two three, one two three, five six seven eight
|
||||
(123) 123-5678~one two three, one two three, five six seven eight
|
||||
123-123-5678-1111~one two three, one two three, five six seven eight, one one one one
|
||||
123-123-5678-1~one two three, one two three, five six seven eight, one
|
||||
@@ -0,0 +1,14 @@
|
||||
01:00~one o'clock
|
||||
01:00 am~one a m
|
||||
01:00 a.m.~one a m
|
||||
01:00 a.m~one a m
|
||||
1 a.m.~one a m
|
||||
23:00~twenty three o'clock
|
||||
1:00 a.m.~one a m
|
||||
1:00a.m.~one a m
|
||||
1.59 p.m.~one fifty nine p m
|
||||
01:59 p.m.~one fifty nine p m
|
||||
1:59 p.m.~one fifty nine p m
|
||||
1:59 p.m. est~one fifty nine p m e s t
|
||||
1:59 p.m.est~one fifty nine p m e s t
|
||||
1:59 p.m. e.s.t~one fifty nine p m e s t
|
||||
@@ -0,0 +1,3 @@
|
||||
dr. Evil~doctor Evil
|
||||
mrs. Norris~misses Norris
|
||||
Dr. Evil~Dr. Evil
|
||||
@@ -0,0 +1,23 @@
|
||||
~
|
||||
es3~e s three
|
||||
x ~x
|
||||
X!~X!
|
||||
2!~two !
|
||||
—~—
|
||||
aabach~aabach
|
||||
aabenraa~aabenraa
|
||||
aabye~aabye
|
||||
aaccessed~aaccessed
|
||||
aach~aach
|
||||
aachen's~aachen's
|
||||
aadri~aadri
|
||||
aafia~aafia
|
||||
aagaard~aagaard
|
||||
aagadu~aagadu
|
||||
aagard~aagard
|
||||
aagathadi~aagathadi
|
||||
aaghart's~aaghart's
|
||||
aalem~aalem
|
||||
a'ali~a'ali
|
||||
aaliyan's~aaliyan's
|
||||
mother-in-law~mother-in-law
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestBoundary:
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_boundary.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(
|
||||
test_input, n_tagged=100, punct_pre_process=False, punct_post_process=False
|
||||
)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestCardinal:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_cardinal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_cardinal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestDate:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_date.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_date.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm_uncased(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
|
||||
normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
cases_uppercased = {"Aug. 8": "august eighth", "8 Aug.": "the eighth of august", "aug. 8": "august eighth"}
|
||||
|
||||
@parameterized.expand(cases_uppercased.items())
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm_cased(self, test_input, expected):
|
||||
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestDecimal:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_decimal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_decimal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestElectronic:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_electronic.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_electronic.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(
|
||||
test_input, n_tagged=100, punct_post_process=False
|
||||
)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,38 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestFraction:
|
||||
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_fraction.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestMeasure:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_measure.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_measure.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestMoney:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_money.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_money.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,33 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, get_test_cases_multiple
|
||||
|
||||
|
||||
class TestNormalizeWithAudio:
|
||||
|
||||
normalizer = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(get_test_cases_multiple('data_text_normalization/test_cases_normalize_with_audio.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, n_tagged=700)
|
||||
assert len(set(pred).intersection(set(expected))) == len(expected), f'pred: {pred}'
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestOrdinal:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_ordinal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_ordinal.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,64 @@
|
||||
#! /bin/sh
|
||||
|
||||
PROJECT_DIR=/workspace/tests
|
||||
|
||||
runtest () {
|
||||
input=$1
|
||||
cd /workspace/sparrowhawk/documentation/grammars
|
||||
|
||||
# read test file
|
||||
while read testcase; do
|
||||
IFS='~' read spoken written <<< $testcase
|
||||
denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1)
|
||||
|
||||
# trim white space
|
||||
written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
|
||||
# input expected actual
|
||||
assertEquals "$spoken" "$written" "$denorm_pred"
|
||||
done < "$input"
|
||||
}
|
||||
|
||||
testITNCardinal() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_cardinal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNDate() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_date.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNDecimal() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_decimal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNOrdinal() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_ordinal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNTime() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_time.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNMeasure() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_measure.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNMoney() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_money.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testITNWhitelist() {
|
||||
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_whitelist.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
# Load shUnit2
|
||||
. $PROJECT_DIR/../shunit2/shunit2
|
||||
@@ -0,0 +1,64 @@
|
||||
#! /bin/sh
|
||||
|
||||
PROJECT_DIR=/workspace/tests
|
||||
|
||||
runtest () {
|
||||
input=$1
|
||||
cd /workspace/sparrowhawk/documentation/grammars
|
||||
|
||||
# read test file
|
||||
while read testcase; do
|
||||
IFS='~' read written spoken <<< $testcase
|
||||
denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1)
|
||||
|
||||
# trim white space
|
||||
spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
|
||||
# input expected actual
|
||||
assertEquals "$written" "$spoken" "$denorm_pred"
|
||||
done < "$input"
|
||||
}
|
||||
|
||||
testTNCardinal() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_cardinal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNDate() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_date.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNDecimal() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_decimal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNOrdinal() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_ordinal.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNTime() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_time.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNMeasure() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_measure.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNMoney() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_money.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
testTNWhitelist() {
|
||||
input=$PROJECT_DIR/data_text_normalization/test_cases_whitelist.txt
|
||||
runtest $input
|
||||
}
|
||||
|
||||
# Load shUnit2
|
||||
. $PROJECT_DIR/../shunit2/shunit2
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestTelephone:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_telephone.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_telephone.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestTime:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_time.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_time.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,65 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestWhitelist:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_whitelist.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='lower_cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_whitelist.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
|
||||
normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
cases_uppercased = {"Dr. Evil": "doctor Evil", "No. 4": "number four", "dr. Evil": "dr. Evil", "no. 4": "no. four"}
|
||||
|
||||
@parameterized.expand(cases_uppercased.items())
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm_cased(self, test_input, expected):
|
||||
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,52 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import pytest
|
||||
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
|
||||
from nemo_text_processing.text_normalization.normalize import Normalizer
|
||||
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
|
||||
from parameterized import parameterized
|
||||
from utils import PYNINI_AVAILABLE, parse_test_case_file
|
||||
|
||||
|
||||
class TestWord:
|
||||
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_word.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_denorm(self, test_input, expected):
|
||||
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
|
||||
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
|
||||
|
||||
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_word.txt'))
|
||||
@pytest.mark.skipif(
|
||||
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
|
||||
)
|
||||
@pytest.mark.run_only_on('CPU')
|
||||
@pytest.mark.unit
|
||||
def test_norm(self, test_input, expected):
|
||||
pred = self.normalizer.normalize(test_input, verbose=False)
|
||||
assert pred == expected
|
||||
pred_non_deterministic = self.normalizer_with_audio.normalize(
|
||||
test_input, n_tagged=100, punct_post_process=False
|
||||
)
|
||||
assert expected in pred_non_deterministic
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
try:
|
||||
import pynini
|
||||
|
||||
PYNINI_AVAILABLE = True
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
PYNINI_AVAILABLE = False
|
||||
|
||||
|
||||
def parse_test_case_file(file_name: str):
|
||||
"""
|
||||
Prepares tests pairs for ITN and TN tests
|
||||
"""
|
||||
test_pairs = []
|
||||
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f:
|
||||
for line in f:
|
||||
spoken, written = line.split('~')
|
||||
test_pairs.append((spoken, written.strip("\n")))
|
||||
return test_pairs
|
||||
|
||||
|
||||
def get_test_cases_multiple(file_name: str = 'data_text_normalization/test_cases_normalize_with_audio.txt'):
|
||||
"""
|
||||
Prepares tests pairs for audio based TN tests
|
||||
"""
|
||||
test_pairs = []
|
||||
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f:
|
||||
written = None
|
||||
normalized_options = []
|
||||
for line in f:
|
||||
if line.startswith('~'):
|
||||
if written:
|
||||
test_pairs.append((written, normalized_options))
|
||||
normalized_options = []
|
||||
written = line.strip().replace('~', '')
|
||||
else:
|
||||
normalized_options.append(line.strip())
|
||||
test_pairs.append((written, normalized_options))
|
||||
return test_pairs
|
||||
Reference in New Issue
Block a user