Introduce code from Rust monorepo. This is a mess

This commit is contained in:
Brandon Thomas
2022-02-28 05:43:54 -05:00
parent 921fb49f9a
commit ba326bfcfd
1058 changed files with 192192 additions and 0 deletions
@@ -0,0 +1,24 @@
nine trillion seven hundred eighty nine billion three hundred eighty two million five hundred thirty six thousand one hundred thirty~9789382536130
two hundred and fifty four~254
one hundred forty seven thousand four hundred fifty one~147451
one million one hundred fifty six thousand one hundred seventy three~1156173
one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961
ninety seven quadrillion eight hundred eight trillion two hundred sixty four billion seven hundred seventy two million seven hundred ninety two thousand five~97808264772792005
seventeen sextillion eight hundred fifty five quintillion thirty six quadrillion six hundred fifty seven trillion seven billion five hundred ninety six million one hundred ten thousand nine hundred forty nine~17855036657007596110949
ten quadrillion ten trillion ten million one hundred thousand ten~10010000010100010
minus twenty five thousand thirty seven~-25037
one quadrillion two hundred sixty four trillion three hundred one billion nine hundred thirty eight million one hundred four~1264301938000104
minus sixty~-60
forty six thousand six hundred sixty four~46664
sixty~60
zero~zero
two million three~2000003
one thousand thirteen~1013
one thousand one~1001
one thousand one hundred~1100
one thousand twenty six~1026
one thousand one hundred twenty six~1126
eighteen million four hundred fifty thousand nine hundred ninety~18450990
eighteen million nine hundred forty thousand seven hundred twenty two~18940722
eighteen million six hundred ninety thousand nine hundred sixteen~18690916
eighteen thousand eight hundred eighty~18880
@@ -0,0 +1,34 @@
july twenty fifth two thousand twelve~july 25 2012
nineteen eighties~1980s
two thousand and twenty~2020
two thousand and nine~2009
the twenty fifth of july twenty twelve~25 july 2012
the twenty fifth of july two thousand twelve~25 july 2012
the twenty second of july twenty twelve~22 july 2012
the fifteenth of january~15 january
the seventeenth of may twenty ten~17 may 2010
january first~january 1
july twenty second two thousand eight~july 22 2008
june thirty~june 30
july twenty fifth twenty twelve~july 25 2012
nineteen seventeen~1917
twenty twelve~2012
march sixteen sixty five~march 1665
sixteen sixty five~1665
july two thousand twelve~july 2012
october nineteen oh five~october 1905
july fifteen o six~july 1506
the twenty fifth of july twenty twelve~25 july 2012
july twenty fifth twenty twelve~july 25 2012
july twenty fifth two thousand twelve~july 25 2012
july one thousand eight hundred seventy six~july 1876
february twenty fifth twenty sixteen~february 25 2016
november twenty fourth twenty fourteen~november 24 2014
nineteen ninety four~1994
two thousand three~2003
one thousand eight~1008
nineteen seventy six~1976
june twentieth twenty fourteen~june 20 2014
nineteen seventy three~1973
nineteen seventy five~1975
eleven fifty five~1155
@@ -0,0 +1,63 @@
five point two million~5.2 million
one hundred sixty four point five eight thousand~164.58 thousand
four hundred million~400 million
fifty billion~50 billion
four hundred five billion~405 billion
four point eight five billion~4.85 billion
one hundred billion~100 billion
one hundred ten billion~110 billion
one hundred thirty two billion~132 billion
one point eight four billion~1.84 billion
one point eight one billion~1.81 billion
one point five nine billion~1.59 billion
one point four five three billion~1.453 billion
one point seven two billion~1.72 billion
one point two five billion~1.25 billion
thirteen billion~13 billion
thirty billion~30 billion
two thousand eight hundred five point eight seven three billion~2805.873 billion
seventy trillion~70 trillion
thirteen million~13 million
eighteen billion~18 billion
four hundred fifty million~450 million
one hundred thirty million~130 million
ten million~10 million
four hundred million~400 million
five million~5 million
five hundred million~500 million
twelve million~12 million
thirteen million~13 million
four million~4 million
forty five million~45 million
fifteen million~15 million
fifteen trillion~15 trillion
fifteen billion~15 billion
two million~2 million
eight million~8 million
point one two o five~.1205
minus sixty point two four zero zero~-60.2400
zero point two six~0.26
point zero two~.02
sixty point two~60.2
eighteen~18
eighteen point eight five~18.85
eighteen point five o~18.50
eighteen point five six~18.56
eighteen point nine~18.9
eighteen point o five~18.05
eighteen point one two~18.12
eighteen point o one~18.01
eighteen point o o o~18.000
eighteen point six~18.6
eighteen point three o o~18.300
eighteen point three six~18.36
eighteen point two five~18.25
eighteen point two two~18.22
eight hundred eighteen point three o three~818.303
eight hundred eight point eight~808.8
eight hundred eight point zero~808.0
eight hundred eighty eight point one~888.1
eight hundred eighty four point three~884.3
eight hundred eighty two point eight~882.8
eight hundred eighty two point zero~882.0
eight hundred forty five point nine four~845.94
@@ -0,0 +1,7 @@
a dot b c at g mail dot com~a.bc@gmail.com
c d f at a b c dot e d u~cdf@abc.edu
a b c at g mail dot a b c~abc@gmail.abc
a b c at a b c dot com~abc@abc.com
a s d f one two three at a b c dot com~asdf123@abc.com
a one b two at a b c dot com~a1b2@abc.com
a b three dot s d d dot three at g mail dot com~ab3.sdd.3@gmail.com
@@ -0,0 +1,112 @@
two hundred meters~200 m
fifty six point three per square kilometer~56.3 /km²
two hundred kilometers per hour~200 km/h
twenty eight kilograms force per square centimeter~28 kgf/cm²
forty two thousand two hundred fifty nine per square meter~42259 /m²
minus two thousand twelve kilo liters~-2012 kl
minus sixty six kilograms~-66 kg
two kilo watt hours~2 kwh
one point o o o o two eight cubic deci meters~1.000028 dm³
seven point five peta bytes~7.5 pb
three hours~3 h
one milli volt~1 mv
two cubic meters~2 m³
ninety grams~90 g
one hundred twenty four point three lumens~124.3 lm
four hundred forty milliliters~440 ml
thirty one thousand four hundred eighty square feet~31480 sq ft
one thousand six hundred hours~1600 h
thirty one thousand four hundred eighty square feet~31480 sq ft
two square miles~2 sq mi
zero point one nine square miles~0.19 sq mi
one thousand five hundred thirty one c c~1531 cc
three hundred micrometers~300 μm
sixty five thousand square kilometers~65000 km²
two miles per hour~2 mph
two hundred forty five miles per hour~245 mph
one hundred fifty c c~150 cc
sixty point two four zero zero kilograms~60.2400 kg
zero feet~0 ft
zero foot~zero foot
two feet~2 ft
twenty foot~20 foot
point two meters~.2 m
two square meters~2 m²
eighteen feet~18 ft
eighteen mega siemens~18 ms
eighteen ounces~18 oz
eighteen point five kilometers~18.5 km
eighteen point five two square kilometers~18.52 km²
eighteen point nine one square kilometers~18.91 km²
eighteen point one four percent~18.14 %
eighteen point one six percent~18.16 %
eighteen point one square kilometers~18.1 km²
eighteen point six percent~18.6 %
eighteen point two two kilometers~18.22 km
eighteen point zero kilometers~18.0 km
eighteen point zero percent~18.0 %
eighteen square kilometers~18 km²
eighteen thousand eight hundred giga watt hours~18800 gwh
eighteen thousand seven hundred hectares~18700 ha
eight hectares~8 ha
eight hundred eighty five astronomical units~885 au
eight hundred eighty hectares~880 ha
eight hundred eighty kilobytes~880 kb
eight hundred eighty kilometers~880 km
eight hundred eighty nine feet~889 ft
eight hundred eighty six kilometers~886 km
eight hundred eighty two megawatts~882 mw
eight hundred feet~800 ft
eight hundred fifty five square kilometers~855 km²
eight hundred fifty megahertz~850 mhz
eight hundred fifty meters~850 m
eight hundred fifty nanometers~850 nm
eight hundred fifty one meters~851 m
eight hundred fifty seven square kilometers~857 km²
eight hundred fifty three meters~853 m
eight hundred fifty three point six meters~853.6 m
eight hundred five point four six square kilometers~805.46 km²
eight hundred forty two point nine meters~842.9 m
eight hundred forty two square kilometers~842 km²
eight hundred gigabytes~800 gb
eight hundred horsepower~800 hp
eight hundred kilograms~800 kg
eight hundred kilo watt hours~800 kwh
eight hundred kilowatts~800 kw
eight hundred megahertz~800 mhz
eight hundred ninety four c c~894 cc
eight hundred ninety kilowatts~890 kw
eight hundred ninety millimeters~890 mm
eight hundred ninety two square kilometers~892 km²
eight hundred seventy horsepower~870 hp
eight hundred seventy meters~870 m
eight hundred sixty kilograms~860 kg
eight hundred sixty kilometers~860 km
eight hundred sixty miles~860 mi
eight hundred sixty six feet~866 ft
eight hundred ten hectares~810 ha
eight hundred ten kilohertz~810 khz
eight hundred thirty eight point two millimeters~838.2 mm
eight hundred thirty five kilometers~835 km
eight hundred thirty kilohertz~830 khz
eight hundred thirty megawatts~830 mw
eight hundred thirty nine kilometers~839 km
eight hundred thirty six meters~836 m
eight hundred twenty feet~820 ft
eight hundred twenty kilometers~820 km
eight hundred twenty meters~820 m
eight hundred twenty one point zero feet~821.0 ft
eight hundred two point eight nine kilometers~802.89 km
eight hundred volts~800 v
eight kilobits~8 kb
eight kilograms~8 kg
eight million two hundred thousand feet~8200000 ft
eight point eight kilometers~8.8 km
eight point eight meters~8.8 m
eight point eight miles~8.8 mi
eight point five centimeters~8.5 cm
eight point five five percent~8.55 %
eight point five megawatts~8.5 mw
eight point five meters~8.5 m
eight point five two percent~8.52 %
eight point four four percent~8.44 %
@@ -0,0 +1,48 @@
two dollars~$2
one cent~$0.01
four united states dollars and sixty nine cents~$4.69
seventy five dollars sixty three~$75.63
twenty nine dollars fifty cents~$29.50
eleven dollars and fifty one cents~$11.51
nine hundred ninety three dollars and ninety two cents~$993.92
four hundred sixty billion won~₩460 billion
thirty billion yen~¥30 billion
two point five billion dollars~$2.5 billion
forty five billion dollars~$45 billion
fifty million dollars~$50 million
fifty billion dollars~$50 billion
zero point two million dollars~$0.2 million
fifteen point two billion dollars~$15.2 billion
one point six nine billion yuan~1.69 billion yuan
one point four three six billion yuan~1.436 billion yuan
four million yuan~4 million yuan
one dollar~$1
fifteen thousand dollars~$15000
one dollars~one dollars
twenty dollar~20 dollar
twenty point five o six dollars~$20.506
point five o six dollars~$.506
eighteen dollars~$18
eighteen million nine hundred twenty five thousand dollars~$18925000
eighteen thousand eight hundred fifty four dollars~$18854
eighteen thousand eight hundred one dollars~$18801
eighteen thousand eight hundred seventy five dollars~$18875
eighteen thousand eighty one dollars~$18081
eighteen thousand fifty two dollars~$18052
eighteen thousand five hundred forty two dollars~$18542
eighteen thousand five hundred nineteen dollars~$18519
eighteen thousand five hundred seventy dollars~$18570
eighteen thousand five hundred seventy eight dollars~$18578
eighteen thousand five hundred sixteen dollars~$18516
eighteen thousand four hundred eighty two dollars~$18482
eighteen thousand four hundred seventy eight dollars~$18478
eighteen thousand four hundred sixty eight dollars~$18468
eighteen thousand nine hundred three dollars~$18903
eighteen thousand nine hundred twenty nine dollars~$18929
eighteen thousand ninety five dollars~$18095
eighteen thousand one hundred seventeen dollars~$18117
eighteen thousand one hundred twenty eight dollars~$18128
eighteen thousand one hundred twenty five dollars~$18125
eighteen thousand one hundred twenty four dollars~$18124
eighteen thousand one hundred twenty nine dollars~$18129
eighteen thousand pounds~£18000
@@ -0,0 +1,33 @@
one hundredth~100th
twenty five thousand one hundred eleventh~25111th
second~2nd
zeroth~0th
first~1st
second~2nd
third~3rd
fourth~4th
eleventh~11th
twelfth~12th
thirteenth~13th
twenty first~21st
twenty third~23rd
one hundred eleventh~111th
one thousandth~1000th
one hundred twenty first~121st
second~2nd
tenth~10th
sixth~6th
third~3rd
nineteenth~19th
third~3rd
twelfth~12th
forty eighth~48th
seventy first~71st
third~3rd
forty second~42nd
seventeenth~17th
twentieth~20th
twenty first~21st
seventh~7th
second~2nd
fifth~5th
@@ -0,0 +1,4 @@
one two three one two three five six seven eight~123-123-5678
zero two three one two three five six seven eight~023-123-5678
o two three one two three five six seven eight~023-123-5678
oh two three one two three five six seven eight~023-123-5678
@@ -0,0 +1,27 @@
eight oclock g m t~08:00 gmt
seven a m e s t~07:00 a.m. est
two p m~02:00 p.m.
two thirty~02:30
three o'clock~03:00
quarter past one~01:15
half past three~03:30
eight fifty one~08:51
eight fifty two~08:52
eight forty~08:40
eight nineteen~08:19
eight o six~08:06
eight thirty eight~08:38
eight thirty two~08:32
eight twenty nine~08:29
eleven fifty five p m~11:55 p.m.
eleven fifty three p m~11:53 p.m.
eleven forty a m~11:40 a.m.
eleven forty five a m~11:45 a.m.
eleven forty p m~11:40 p.m.
eleven forty six a m~11:46 a.m.
eleven o six p m~11:06 p.m.
eleven thirteen a m~11:13 a.m.
half past twelve~12:30
quarter past one~01:15
quarter to one~12:45
quarter to twelve~11:45
@@ -0,0 +1,8 @@
doctor dao~dr. dao
miss smith~ms. smith
misses smith~mrs. smith
mister dao~mr. dao
saint george~st. george
i like for example ice cream~i like e.g. ice cream
s and p five hundred~s&p 500
seven eleven stores~7-eleven stores
@@ -0,0 +1,50 @@
~
e s three~es3
yahoo!~yahoo!
twenty!~20 !
x ~x
—~—
aaa~aaa
aabach~aabach
aabenraa~aabenraa
aabye~aabye
aaccessed~aaccessed
aach~aach
aachen's~aachen's
aadri~aadri
aafia~aafia
aagaard~aagaard
aagadu~aagadu
aagard~aagard
aagathadi~aagathadi
aaghart's~aaghart's
aagnes~aagnes
aagomoni~aagomoni
aagon~aagon
aagoo~aagoo
aagot~aagot
aahar~aahar
aahh~aahh
aahperd~aahperd
aaibinterstate~aaibinterstate
aajab~aajab
aakasa~aakasa
aakervik~aakervik
aakirkeby~aakirkeby
aalam~aalam
aalbaek~aalbaek
aaldiu~aaldiu
aalem~aalem
a'ali~a'ali
aalilaassamthey~aalilaassamthey
aalin~aalin
aaliyan~aaliyan
aaliyan's~aaliyan's
aamadu~aamadu
aamara~aamara
aambala~aambala
aamera~aamera
aamer's~aamer's
aamina~aamina
aaminah~aaminah
aamjiwnaang~aamjiwnaang
@@ -0,0 +1,7 @@
1~one
1~one
1, ~one ,
1!!!!~one ! ! ! !
(1)Hello~(1)Hello
!1~! one
1!hello~1!hello
@@ -0,0 +1,18 @@
1~one
2~two
-2~minus two
3~three
123~one hundred twenty three
13,000~thirteen thousand
13,000~thirteen thousand
123,123,000~one hundred twenty three million one hundred twenty three thousand
123,000,012~one hundred twenty three million twelve
t-0t25d12-f~t oh t two five d one two f
133-ABC~one hundred thirty three ABC
B2A23C~B two A two three C
25d08A~two five d oh eight A
C24~C two four
W2s~W two s
1-4-a-b-1-5~one four a b one five
b-c-1-5-b-s-b~b c one five b s b
1/f-4s~one f four s
@@ -0,0 +1,32 @@
july 25 2012~july twenty fifth twenty twelve
jul 25 2012~july twenty fifth twenty twelve
1980s~nineteen eighties
1980 s~nineteen eighties
25 july 2012~the twenty fifth of july twenty twelve
25 jul 2012~the twenty fifth of july twenty twelve
22 july 2012~the twenty second of july twenty twelve
15 january~the fifteenth of january
17 may 2010~the seventeenth of may twenty ten
january 1~january first
june 30~june thirtieth
2012~twenty twelve
july 1506~july fifteen o six
july 1876~july eighteen seventy six
1994~nineteen ninety four
1976~nineteen seventy six
june 20 2014~june twentieth twenty fourteen
1973~nineteen seventy three
1975~nineteen seventy five
1155~eleven fifty five
2006-08-05~august fifth two thousand six
august 23, 2002~august twenty third two thousand two
20 january~the twentieth of january
2016-11-03~november third twenty sixteen
2016/07/03~july third twenty sixteen
1980s~nineteen eighties
01.10.2010~january tenth twenty ten
10/06/2005~october sixth two thousand five
1910s~nineteen tens
25 sept.~the twenty fifth of september
1000~one thousand
SEPT. 15TH~september fifteenth
@@ -0,0 +1,8 @@
.1665~point one six six five
2.0~two point zero
2.00~two point o o
2.050~two point o five o
100 million~one hundred million
0.1 billion~zero point one billion
.1 trillion~point one trillion
-0.1~minus zero point one
@@ -0,0 +1,10 @@
a.bc@gmail.com~a dot b c at g mail dot com
cdf@abc.edu~c d f at a b c dot e d u
abc@gmail.abc~a b c at g mail dot a b c
abc@abc.com~a b c at a b c dot com
asdf123@abc.com~a s d f one two three at a b c dot com
a1b2@abc.com~a one b two at a b c dot com
ab3.sdd.3@gmail.com~a b three dot s d d dot three at g mail dot com
email is abc1@gmail.com~email is a b c one at g mail dot com
test .b@gmail.com~test . b at g mail dot com
email is a-1.b3_c&-d@gma4i-l.com test~email is a dash one dot b three underscore c ampersand dash d at g m a four i dash l dot com test
@@ -0,0 +1,11 @@
1/2007~one two thousand seventh
12639/12640~twelve thousand six hundred thirty nine twelve thousand six hundred fortieths
2/4~two quarters
1/4~one quarter
31/32~thirty one thirty seconds
22/3~twenty two thirds
1/3~one third
142/1~one hundred forty two over one
1/2~one half
2 1/2~two and a half
1795 / 1805~one thousand seven hundred ninety five one thousand eight hundred fifths
@@ -0,0 +1,15 @@
12kg~twelve kilograms
12/kg~twelve per kilogram
12kg/kg~twelve kilograms per kilogram
1 mbps~one megabit per second
3 mbps~three megabits per second
3 cc/s~three c c per second
100 million kg~one hundred million kilograms
a 4-kilogram bag~a four kilogram bag
55-millimeters long~fifty five millimeters long
a 123.2-millimeters long~a one hundred twenty three point two millimeters long
covid-19.5~covid nineteen point five
covid-19~covid nineteen
a 4-kilogram bag~a four kilogram bag
7.2-millimeter bullet~seven point two millimeter bullet
4 1/2 lbs~four and a half pounds
@@ -0,0 +1,38 @@
$2~two dollars
₩460 billion~four hundred sixty billion won
¥30 billion~thirty billion yen
$45 billion~forty five billion dollars
$0.2 million~zero point two million dollars
$15.2 billion~fifteen point two billion dollars
1.436 billion yuan~one point four three six billion yuan
4 million yuan~four million yuan
$15000~fifteen thousand dollars
$1~one dollar
$20~twenty dollars
$20.506~twenty point five o six dollars
$.506~point five o six dollars
$18~eighteen dollars
$18925000~eighteen million nine hundred twenty five thousand dollars
$18,925,000~eighteen million nine hundred twenty five thousand dollars
$18854~eighteen thousand eight hundred fifty four dollars
$18801~eighteen thousand eight hundred one dollars
$18875~eighteen thousand eight hundred seventy five dollars
$18081~eighteen thousand eighty one dollars
$18052~eighteen thousand fifty two dollars
$18542~eighteen thousand five hundred forty two dollars
$18519~eighteen thousand five hundred nineteen dollars
$18570~eighteen thousand five hundred seventy dollars
$18578~eighteen thousand five hundred seventy eight dollars
$18516~eighteen thousand five hundred sixteen dollars
$18482~eighteen thousand four hundred eighty two dollars
$18478~eighteen thousand four hundred seventy eight dollars
$18468~eighteen thousand four hundred sixty eight dollars
$18903~eighteen thousand nine hundred three dollars
$18929~eighteen thousand nine hundred twenty nine dollars
$18095~eighteen thousand ninety five dollars
$18117~eighteen thousand one hundred seventeen dollars
$18128~eighteen thousand one hundred twenty eight dollars
$18125~eighteen thousand one hundred twenty five dollars
$18124~eighteen thousand one hundred twenty four dollars
$18129~eighteen thousand one hundred twenty nine dollars
£18000~eighteen thousand pounds
@@ -0,0 +1,107 @@
~123
one twenty three
one two three
one hundred twenty three
one hundred and twenty three
~$123
one twenty three dollars
one two three dollars
one hundred twenty three dollars
one hundred and twenty three dollars
~$123.2
one twenty three dollars and two cents
one two three dollars two cents
one hundred twenty three dollars and two cents
one hundred and twenty three point two dollars
~1.24
one point two four
one two four
one point twenty four
one twenty four
~$1.21
one dollar and twenty one cents
one dollar twenty one cents
one point two one dollars
one twenty one dollars
~£1.00
one pound
one point o o pounds
~t-0t25d12-f
t zero t twenty five d one two f
t zero t twenty five d twelve f
t zero t two five d one two f
t zero t two five d twelve f
t oh t two five d one two f
t oh t twenty five d one two f
t oh t twenty five d twelve f
t oh t two five d twelve f
~133-A
one hundred thirty three A
~B2A23C
B two A twenty three C
~25d08A
twenty five d zero eight A
~C24
C twenty four
~It seemed to her that the jacket Oswald wore was darker than Commission Exhibit No. 162.
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one sixty two.
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred sixty two.
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one hundred and sixty two.
It seemed to her that the jacket Oswald wore was darker than Commission Exhibit number one six two.
~"Father, let this cup pass." He prayed--was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord:-- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim."
"Father, let this cup pass." He prayed -- was heard. What cup was it that passed away from him? Sure not the death-cup, now filled to the brim! There was no quailing in the awful word; He still was king of kings, of lords the lord: -- He feared lest, in the suffering waste and grim, His faith might grow too faint and sickly dim."
~1970-2010
nineteen seventy to twenty ten
one thousand nine seventy to two thousand ten
one thousand nine hundred and seventy to two thousand ten
one thousand nine hundred and seventy to twenty ten
nineteen seventy twenty ten
one thousand nine seventy two thousand ten
one thousand nine hundred and seventy two thousand ten
one thousand nine hundred and seventy twenty ten
from nineteen seventy to twenty ten
from one thousand nine seventy to two thousand ten
from one thousand nine hundred and seventy to two thousand ten
from one thousand nine hundred and seventy to twenty ten
from nineteen seventy twenty ten
from one thousand nine seventy two thousand ten
from one thousand nine hundred and seventy two thousand ten
from one thousand nine hundred and seventy twenty ten
~W26s
W twenty sixes
W twenty six s
~5-3-Ws
five three W s
five three Ws
~401-ks
four oh one kays
four oh one k s
four hundred one k s
~The box was 25 x 7 m.
The box was twenty five by seven meters.
The box was two five by seven meters.
The box was two five by seven m.
The box was twenty five by seven m.
~4567
four thousand five hundred sixty seven
four five six seven
four thousand five sixty seven
forty five sixty seven
four thousand five hundred and sixty seven
~This example number 15,000 can be a very long one, and can fail to produce valid normalization for a such an easy number like 10,125 or dollar value $5349.01, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, 452.
This example number fifteen thousand can be a very long one, and can fail to produce valid normalization for a such an easy number like ten thousand one hundred twenty five or dollar value five thousand three hundred forty nine dollars and one cent, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, and can fail to terminate, four five two.
~$1.01
one dollar and one cent
one dollar one cent
one dollar and zero one cents
~$17.31
seventeen dollars and thirty one cent
seventeen dollars and thirty one cents
seventeen point three one dollars
~25.]
two five.]
twenty five.]
~Francis I--test
Francis the first -- test
Francis one -- test
Francis first -- test
@@ -0,0 +1,6 @@
0th~zeroth
11th~eleventh
23rd~twenty third
1000th~one thousandth
1,000th~one thousandth
1000000th~one millionth
@@ -0,0 +1,9 @@
+1 123-123-5678~one, one two three, one two three, five six seven eight
123-123-5678~one two three, one two three, five six seven eight
+1-123-123-5678~one, one two three, one two three, five six seven eight
+1 (123)-123-5678~one, one two three, one two three, five six seven eight
+1(123) 123-5678~one, one two three, one two three, five six seven eight
(123)-123-5678~one two three, one two three, five six seven eight
(123) 123-5678~one two three, one two three, five six seven eight
123-123-5678-1111~one two three, one two three, five six seven eight, one one one one
123-123-5678-1~one two three, one two three, five six seven eight, one
@@ -0,0 +1,14 @@
01:00~one o'clock
01:00 am~one a m
01:00 a.m.~one a m
01:00 a.m~one a m
1 a.m.~one a m
23:00~twenty three o'clock
1:00 a.m.~one a m
1:00a.m.~one a m
1.59 p.m.~one fifty nine p m
01:59 p.m.~one fifty nine p m
1:59 p.m.~one fifty nine p m
1:59 p.m. est~one fifty nine p m e s t
1:59 p.m.est~one fifty nine p m e s t
1:59 p.m. e.s.t~one fifty nine p m e s t
@@ -0,0 +1,3 @@
dr. Evil~doctor Evil
mrs. Norris~misses Norris
Dr. Evil~Dr. Evil
@@ -0,0 +1,23 @@
~
es3~e s three
x ~x
X!~X!
2!~two !
—~—
aabach~aabach
aabenraa~aabenraa
aabye~aabye
aaccessed~aaccessed
aach~aach
aachen's~aachen's
aadri~aadri
aafia~aafia
aagaard~aagaard
aagadu~aagadu
aagard~aagard
aagathadi~aagathadi
aaghart's~aaghart's
aalem~aalem
a'ali~a'ali
aaliyan's~aaliyan's
mother-in-law~mother-in-law
@@ -0,0 +1,39 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestBoundary:
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_boundary.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(
test_input, n_tagged=100, punct_pre_process=False, punct_post_process=False
)
assert expected in pred_non_deterministic
@@ -0,0 +1,49 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestCardinal:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_cardinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_cardinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,64 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestDate:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_date.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_date.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_uncased(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
cases_uppercased = {"Aug. 8": "august eighth", "8 Aug.": "the eighth of august", "aug. 8": "august eighth"}
@parameterized.expand(cases_uppercased.items())
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_cased(self, test_input, expected):
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,49 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestDecimal:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_decimal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_decimal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,51 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestElectronic:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_electronic.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_electronic.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(
test_input, n_tagged=100, punct_post_process=False
)
assert expected in pred_non_deterministic
@@ -0,0 +1,38 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestFraction:
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_fraction.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,50 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestMeasure:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_measure.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case="cased") if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_measure.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,50 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestMoney:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_money.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_money.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,33 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, get_test_cases_multiple
class TestNormalizeWithAudio:
normalizer = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(get_test_cases_multiple('data_text_normalization/test_cases_normalize_with_audio.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, n_tagged=700)
assert len(set(pred).intersection(set(expected))) == len(expected), f'pred: {pred}'
@@ -0,0 +1,50 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestOrdinal:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_ordinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_ordinal.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,64 @@
#! /bin/sh
PROJECT_DIR=/workspace/tests
runtest () {
input=$1
cd /workspace/sparrowhawk/documentation/grammars
# read test file
while read testcase; do
IFS='~' read spoken written <<< $testcase
denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1)
# trim white space
written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
# input expected actual
assertEquals "$spoken" "$written" "$denorm_pred"
done < "$input"
}
testITNCardinal() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_cardinal.txt
runtest $input
}
testITNDate() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_date.txt
runtest $input
}
testITNDecimal() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_decimal.txt
runtest $input
}
testITNOrdinal() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_ordinal.txt
runtest $input
}
testITNTime() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_time.txt
runtest $input
}
testITNMeasure() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_measure.txt
runtest $input
}
testITNMoney() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_money.txt
runtest $input
}
testITNWhitelist() {
input=$PROJECT_DIR/data_inverse_text_normalization/test_cases_whitelist.txt
runtest $input
}
# Load shUnit2
. $PROJECT_DIR/../shunit2/shunit2
@@ -0,0 +1,64 @@
#! /bin/sh
PROJECT_DIR=/workspace/tests
runtest () {
input=$1
cd /workspace/sparrowhawk/documentation/grammars
# read test file
while read testcase; do
IFS='~' read written spoken <<< $testcase
denorm_pred=$(echo $written | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1)
# trim white space
spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
# input expected actual
assertEquals "$written" "$spoken" "$denorm_pred"
done < "$input"
}
testTNCardinal() {
input=$PROJECT_DIR/data_text_normalization/test_cases_cardinal.txt
runtest $input
}
testTNDate() {
input=$PROJECT_DIR/data_text_normalization/test_cases_date.txt
runtest $input
}
testTNDecimal() {
input=$PROJECT_DIR/data_text_normalization/test_cases_decimal.txt
runtest $input
}
testTNOrdinal() {
input=$PROJECT_DIR/data_text_normalization/test_cases_ordinal.txt
runtest $input
}
testTNTime() {
input=$PROJECT_DIR/data_text_normalization/test_cases_time.txt
runtest $input
}
testTNMeasure() {
input=$PROJECT_DIR/data_text_normalization/test_cases_measure.txt
runtest $input
}
testTNMoney() {
input=$PROJECT_DIR/data_text_normalization/test_cases_money.txt
runtest $input
}
testTNWhitelist() {
input=$PROJECT_DIR/data_text_normalization/test_cases_whitelist.txt
runtest $input
}
# Load shUnit2
. $PROJECT_DIR/../shunit2/shunit2
@@ -0,0 +1,50 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestTelephone:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_telephone.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_telephone.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,49 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestTime:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_time.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_time.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,65 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestWhitelist:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='lower_cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_whitelist.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
normalizer_uppercased = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
cases_uppercased = {"Dr. Evil": "doctor Evil", "No. 4": "number four", "dr. Evil": "dr. Evil", "no. 4": "no. four"}
@parameterized.expand(cases_uppercased.items())
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm_cased(self, test_input, expected):
pred = self.normalizer_uppercased.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(test_input, n_tagged=100)
assert expected in pred_non_deterministic
@@ -0,0 +1,52 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer
from nemo_text_processing.text_normalization.normalize import Normalizer
from nemo_text_processing.text_normalization.normalize_with_audio import NormalizerWithAudio
from parameterized import parameterized
from utils import PYNINI_AVAILABLE, parse_test_case_file
class TestWord:
inverse_normalizer = InverseNormalizer() if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_inverse_text_normalization/test_cases_word.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_denorm(self, test_input, expected):
pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False)
assert pred == expected
normalizer = Normalizer(input_case='cased') if PYNINI_AVAILABLE else None
normalizer_with_audio = NormalizerWithAudio(input_case='cased') if PYNINI_AVAILABLE else None
@parameterized.expand(parse_test_case_file('data_text_normalization/test_cases_word.txt'))
@pytest.mark.skipif(
not PYNINI_AVAILABLE, reason="`pynini` not installed, please install via nemo_text_processing/setup.sh"
)
@pytest.mark.run_only_on('CPU')
@pytest.mark.unit
def test_norm(self, test_input, expected):
pred = self.normalizer.normalize(test_input, verbose=False)
assert pred == expected
pred_non_deterministic = self.normalizer_with_audio.normalize(
test_input, n_tagged=100, punct_post_process=False
)
assert expected in pred_non_deterministic
@@ -0,0 +1,54 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
try:
import pynini
PYNINI_AVAILABLE = True
except (ImportError, ModuleNotFoundError):
PYNINI_AVAILABLE = False
def parse_test_case_file(file_name: str):
"""
Prepares tests pairs for ITN and TN tests
"""
test_pairs = []
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f:
for line in f:
spoken, written = line.split('~')
test_pairs.append((spoken, written.strip("\n")))
return test_pairs
def get_test_cases_multiple(file_name: str = 'data_text_normalization/test_cases_normalize_with_audio.txt'):
"""
Prepares tests pairs for audio based TN tests
"""
test_pairs = []
with open(os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name, 'r') as f:
written = None
normalized_options = []
for line in f:
if line.startswith('~'):
if written:
test_pairs.append((written, normalized_options))
normalized_options = []
written = line.strip().replace('~', '')
else:
normalized_options.append(line.strip())
test_pairs.append((written, normalized_options))
return test_pairs