mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-19 07:54:56 +00:00
Introduce code from Rust monorepo. This is a mess
This commit is contained in:
Brandon Thomas
@@ -0,0 +1,13 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
@@ -0,0 +1,69 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copyright 2018-2020 William Falcon
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from .pl_utils import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LossInput:
|
||||
"""
|
||||
The input for ``nemo.collections.common.metrics.GlobalAverageLossMetric`` metric tests.
|
||||
|
||||
Args:
|
||||
loss_sum_or_avg: a one dimensional float tensor which contains losses for averaging. Each element is either a
|
||||
sum or mean of several losses depending on the parameter ``take_avg_loss`` of the
|
||||
``nemo.collections.common.metrics.GlobalAverageLossMetric`` class.
|
||||
num_measurements: a one dimensional integer tensor which contains number of measurements which sums or average
|
||||
values are in ``loss_sum_or_avg``.
|
||||
"""
|
||||
|
||||
loss_sum_or_avg: torch.Tensor
|
||||
num_measurements: torch.Tensor
|
||||
|
||||
|
||||
NO_ZERO_NUM_MEASUREMENTS = LossInput(
|
||||
loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0, num_measurements=torch.randint(1, 100, (NUM_BATCHES,)),
|
||||
)
|
||||
|
||||
SOME_NUM_MEASUREMENTS_ARE_ZERO = LossInput(
|
||||
loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0,
|
||||
num_measurements=torch.cat(
|
||||
(
|
||||
torch.randint(1, 100, (NUM_BATCHES // 2,), dtype=torch.int32),
|
||||
torch.zeros(NUM_BATCHES - NUM_BATCHES // 2, dtype=torch.int32),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
ALL_NUM_MEASUREMENTS_ARE_ZERO = LossInput(
|
||||
loss_sum_or_avg=torch.rand(NUM_BATCHES) * 2.0 - 1.0, num_measurements=torch.zeros(NUM_BATCHES, dtype=torch.int32),
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copyright 2018-2020 William Falcon
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
import torch
|
||||
|
||||
from .pl_utils import BATCH_SIZE, NUM_BATCHES, NUM_CLASSES
|
||||
|
||||
Input = namedtuple('Input', ["probs", "logits"])
|
||||
|
||||
|
||||
ONLY_PROBS = Input(probs=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES), logits=None)
|
||||
|
||||
ONLY_LOGITS1 = Input(probs=None, logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES))
|
||||
|
||||
ONLY_LOGITS100 = Input(probs=None, logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES) * 200 - 100)
|
||||
|
||||
PROBS_AND_LOGITS = Input(
|
||||
probs=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES),
|
||||
logits=torch.rand(NUM_BATCHES, BATCH_SIZE, NUM_CLASSES) * 200 - 100,
|
||||
)
|
||||
|
||||
NO_PROBS_NO_LOGITS = Input(probs=None, logits=None)
|
||||
@@ -0,0 +1,557 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copyright 2018-2020 William Falcon
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
from functools import partial
|
||||
from typing import Callable, Optional
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from scipy.stats import entropy
|
||||
from torch.distributions.utils import logits_to_probs
|
||||
from torch.multiprocessing import Pool, set_start_method
|
||||
from torchmetrics import Metric
|
||||
|
||||
from nemo.collections.common.metrics import GlobalAverageLossMetric, Perplexity
|
||||
|
||||
NUM_PROCESSES = 2
|
||||
NUM_BATCHES = 10
|
||||
BATCH_SIZE = 16
|
||||
NUM_CLASSES = 5
|
||||
EXTRA_DIM = 3
|
||||
THRESHOLD = 0.5
|
||||
|
||||
|
||||
def setup_ddp(rank, world_size):
|
||||
""" Setup ddp enviroment """
|
||||
os.environ["MASTER_ADDR"] = 'localhost'
|
||||
os.environ['MASTER_PORT'] = '8088'
|
||||
|
||||
if torch.distributed.is_available() and sys.platform not in ['win32', 'cygwin']:
|
||||
torch.distributed.init_process_group("gloo", rank=rank, world_size=world_size)
|
||||
|
||||
|
||||
def _class_test(
|
||||
rank: int,
|
||||
worldsize: int,
|
||||
preds: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
metric_class: Metric,
|
||||
sk_metric: Callable,
|
||||
dist_sync_on_step: bool,
|
||||
metric_args: dict = {},
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
atol: float = 1e-8,
|
||||
):
|
||||
""" Utility function doing the actual comparison between lightning class metric
|
||||
and reference metric.
|
||||
Args:
|
||||
rank: rank of current process
|
||||
worldsize: number of processes
|
||||
preds: torch tensor with predictions
|
||||
target: torch tensor with targets
|
||||
metric_class: lightning metric class that should be tested
|
||||
sk_metric: callable function that is used for comparison
|
||||
dist_sync_on_step: bool, if true will synchronize metric state across
|
||||
processes at each ``forward()``
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
check_dist_sync_on_step: bool, if true will check if the metric is also correctly
|
||||
calculated per batch per device (and not just at the end)
|
||||
check_batch: bool, if true will check if the metric is also correctly
|
||||
calculated across devices for each batch (and not just at the end)
|
||||
"""
|
||||
# Instanciate lightning metric
|
||||
metric = metric_class(compute_on_step=True, dist_sync_on_step=dist_sync_on_step, **metric_args)
|
||||
|
||||
# verify metrics work after being loaded from pickled state
|
||||
pickled_metric = pickle.dumps(metric)
|
||||
metric = pickle.loads(pickled_metric)
|
||||
|
||||
for i in range(rank, NUM_BATCHES, worldsize):
|
||||
batch_result = metric(preds[i], target[i])
|
||||
|
||||
if metric.dist_sync_on_step:
|
||||
if rank == 0:
|
||||
ddp_preds = torch.stack([preds[i + r] for r in range(worldsize)])
|
||||
ddp_target = torch.stack([target[i + r] for r in range(worldsize)])
|
||||
sk_batch_result = sk_metric(ddp_preds, ddp_target)
|
||||
# assert for dist_sync_on_step
|
||||
if check_dist_sync_on_step:
|
||||
assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol)
|
||||
else:
|
||||
sk_batch_result = sk_metric(preds[i], target[i])
|
||||
# assert for batch
|
||||
if check_batch:
|
||||
assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol)
|
||||
|
||||
# check on all batches on all ranks
|
||||
result = metric.compute()
|
||||
assert isinstance(result, torch.Tensor)
|
||||
|
||||
total_preds = torch.stack([preds[i] for i in range(NUM_BATCHES)])
|
||||
total_target = torch.stack([target[i] for i in range(NUM_BATCHES)])
|
||||
sk_result = sk_metric(total_preds, total_target)
|
||||
|
||||
# assert after aggregation
|
||||
assert np.allclose(result.numpy(), sk_result, atol=atol)
|
||||
|
||||
|
||||
def _functional_test(
|
||||
preds: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
metric_functional: Callable,
|
||||
sk_metric: Callable,
|
||||
metric_args: dict = {},
|
||||
atol: float = 1e-8,
|
||||
):
|
||||
""" Utility function doing the actual comparison between lightning functional metric
|
||||
and reference metric.
|
||||
Args:
|
||||
preds: torch tensor with predictions
|
||||
target: torch tensor with targets
|
||||
metric_functional: lightning metric functional that should be tested
|
||||
sk_metric: callable function that is used for comparison
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
"""
|
||||
metric = partial(metric_functional, **metric_args)
|
||||
|
||||
for i in range(NUM_BATCHES):
|
||||
lightning_result = metric(preds[i], target[i])
|
||||
sk_result = sk_metric(preds[i], target[i])
|
||||
|
||||
# assert its the same
|
||||
assert np.allclose(lightning_result.numpy(), sk_result, atol=atol)
|
||||
|
||||
|
||||
class MetricTester:
|
||||
""" Class used for efficiently run alot of parametrized tests in ddp mode.
|
||||
Makes sure that ddp is only setup once and that pool of processes are
|
||||
used for all tests.
|
||||
All tests should subclass from this and implement a new method called
|
||||
`test_metric_name`
|
||||
where the method `self.run_metric_test` is called inside.
|
||||
"""
|
||||
|
||||
atol = 1e-8
|
||||
|
||||
def setup_class(self):
|
||||
""" Setup the metric class. This will spawn the pool of workers that are
|
||||
used for metric testing and setup_ddp
|
||||
"""
|
||||
try:
|
||||
set_start_method('spawn')
|
||||
except RuntimeError:
|
||||
pass
|
||||
self.poolSize = NUM_PROCESSES
|
||||
self.pool = Pool(processes=self.poolSize)
|
||||
self.pool.starmap(setup_ddp, [(rank, self.poolSize) for rank in range(self.poolSize)])
|
||||
|
||||
def teardown_class(self):
|
||||
""" Close pool of workers """
|
||||
self.pool.close()
|
||||
self.pool.join()
|
||||
|
||||
def run_functional_metric_test(
|
||||
self,
|
||||
preds: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
metric_functional: Callable,
|
||||
sk_metric: Callable,
|
||||
metric_args: dict = {},
|
||||
):
|
||||
""" Main method that should be used for testing functions. Call this inside
|
||||
testing method
|
||||
Args:
|
||||
preds: torch tensor with predictions
|
||||
target: torch tensor with targets
|
||||
metric_functional: lightning metric class that should be tested
|
||||
sk_metric: callable function that is used for comparison
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
"""
|
||||
_functional_test(
|
||||
preds=preds,
|
||||
target=target,
|
||||
metric_functional=metric_functional,
|
||||
sk_metric=sk_metric,
|
||||
metric_args=metric_args,
|
||||
atol=self.atol,
|
||||
)
|
||||
|
||||
def run_class_metric_test(
|
||||
self,
|
||||
ddp: bool,
|
||||
preds: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
metric_class: Metric,
|
||||
sk_metric: Callable,
|
||||
dist_sync_on_step: bool,
|
||||
metric_args: dict = {},
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
):
|
||||
""" Main method that should be used for testing class. Call this inside testing
|
||||
methods.
|
||||
Args:
|
||||
ddp: bool, if running in ddp mode or not
|
||||
preds: torch tensor with predictions
|
||||
target: torch tensor with targets
|
||||
metric_class: lightning metric class that should be tested
|
||||
sk_metric: callable function that is used for comparison
|
||||
dist_sync_on_step: bool, if true will synchronize metric state across
|
||||
processes at each ``forward()``
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
check_dist_sync_on_step: bool, if true will check if the metric is also correctly
|
||||
calculated per batch per device (and not just at the end)
|
||||
check_batch: bool, if true will check if the metric is also correctly
|
||||
calculated across devices for each batch (and not just at the end)
|
||||
"""
|
||||
if ddp:
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("DDP not supported on windows")
|
||||
|
||||
self.pool.starmap(
|
||||
partial(
|
||||
_class_test,
|
||||
preds=preds,
|
||||
target=target,
|
||||
metric_class=metric_class,
|
||||
sk_metric=sk_metric,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
metric_args=metric_args,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
),
|
||||
[(rank, self.poolSize) for rank in range(self.poolSize)],
|
||||
)
|
||||
else:
|
||||
_class_test(
|
||||
0,
|
||||
1,
|
||||
preds=preds,
|
||||
target=target,
|
||||
metric_class=metric_class,
|
||||
sk_metric=sk_metric,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
metric_args=metric_args,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
)
|
||||
|
||||
|
||||
def reference_perplexity_func(probs):
|
||||
ent = entropy(probs, axis=-1)
|
||||
ppl = np.exp(ent)
|
||||
return ppl.mean()
|
||||
|
||||
|
||||
def _perplexity_class_test(
|
||||
rank: int,
|
||||
worldsize: int,
|
||||
probs: Optional[torch.Tensor],
|
||||
logits: Optional[torch.Tensor],
|
||||
dist_sync_on_step: bool,
|
||||
metric_args: dict = {},
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
atol: float = 1e-8,
|
||||
):
|
||||
""" Utility function doing the actual comparison between lightning class metric
|
||||
and reference metric.
|
||||
Args:
|
||||
rank: rank of current process
|
||||
worldsize: number of processes
|
||||
probs: torch tensor with probabilities
|
||||
logits: torch tensor with logits. The function checks ``probs`` and ``logits are mutually exclusive for
|
||||
``Perplexity`` metric.
|
||||
dist_sync_on_step: bool, if true will synchronize metric state across
|
||||
processes at each ``forward()``
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
check_dist_sync_on_step: bool, if true will check if the metric is also correctly
|
||||
calculated per batch per device (and not just at the end)
|
||||
check_batch: bool, if true will check if the metric is also correctly
|
||||
calculated across devices for each batch (and not just at the end)
|
||||
"""
|
||||
# Instanciate lightning metric
|
||||
perplexity = Perplexity(compute_on_step=True, dist_sync_on_step=dist_sync_on_step, **metric_args)
|
||||
if (probs is None) == (logits is None):
|
||||
with pytest.raises(ValueError):
|
||||
perplexity(probs, logits)
|
||||
return
|
||||
|
||||
# verify perplexity works after being loaded from pickled state
|
||||
pickled_metric = pickle.dumps(perplexity)
|
||||
perplexity = pickle.loads(pickled_metric)
|
||||
|
||||
for i in range(rank, NUM_BATCHES, worldsize):
|
||||
batch_result = perplexity(None if probs is None else probs[i], None if logits is None else logits[i])
|
||||
|
||||
if perplexity.dist_sync_on_step:
|
||||
if rank == 0:
|
||||
if probs is not None:
|
||||
ddp_probs = torch.stack([probs[i + r] for r in range(worldsize)])
|
||||
else:
|
||||
ddp_logits = torch.stack([logits[i + r] for r in range(worldsize)])
|
||||
ddp_probs = logits_to_probs(ddp_logits, is_binary=False)
|
||||
sk_batch_result = reference_perplexity_func(ddp_probs)
|
||||
# assert for dist_sync_on_step
|
||||
if check_dist_sync_on_step:
|
||||
assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol)
|
||||
else:
|
||||
if probs is None:
|
||||
p = logits_to_probs(logits[i], is_binary=False)
|
||||
else:
|
||||
p = probs[i]
|
||||
sk_batch_result = reference_perplexity_func(p)
|
||||
# assert for batch
|
||||
if check_batch:
|
||||
assert np.allclose(batch_result.numpy(), sk_batch_result, atol=atol)
|
||||
|
||||
assert (probs is None) != (logits is None)
|
||||
# check on all batches on all ranks
|
||||
result = perplexity.compute()
|
||||
assert isinstance(result, torch.Tensor)
|
||||
|
||||
if probs is None:
|
||||
probs = logits_to_probs(logits, is_binary=False)
|
||||
sk_result = reference_perplexity_func(probs)
|
||||
|
||||
# assert after aggregation
|
||||
assert np.allclose(result.numpy(), sk_result, atol=atol)
|
||||
|
||||
|
||||
class PerplexityTester(MetricTester):
|
||||
def run_class_perplexity_test(
|
||||
self,
|
||||
ddp: bool,
|
||||
probs: Optional[torch.Tensor],
|
||||
logits: Optional[torch.Tensor],
|
||||
dist_sync_on_step: bool,
|
||||
metric_args: dict = {},
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
):
|
||||
""" Main method that should be used for testing class. Call this inside testing
|
||||
methods.
|
||||
Args:
|
||||
ddp: bool, if running in ddp mode or not
|
||||
probs: torch tensor with probabilities.
|
||||
logits: torch tensor with logits. This test checks that probs and logits are mutually exclusive for
|
||||
``Perplexity`` metric.
|
||||
dist_sync_on_step: bool, if true will synchronize metric state across
|
||||
processes at each ``forward()``
|
||||
metric_args: dict with additional arguments used for class initialization
|
||||
check_dist_sync_on_step: bool, if true will check if the metric is also correctly
|
||||
calculated per batch per device (and not just at the end)
|
||||
check_batch: bool, if true will check if the metric is also correctly
|
||||
calculated across devices for each batch (and not just at the end)
|
||||
"""
|
||||
if ddp:
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("DDP not supported on windows")
|
||||
|
||||
self.pool.starmap(
|
||||
partial(
|
||||
_perplexity_class_test,
|
||||
probs=probs,
|
||||
logits=logits,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
metric_args=metric_args,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
),
|
||||
[(rank, self.poolSize) for rank in range(self.poolSize)],
|
||||
)
|
||||
else:
|
||||
_perplexity_class_test(
|
||||
0,
|
||||
1,
|
||||
probs=probs,
|
||||
logits=logits,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
metric_args=metric_args,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
)
|
||||
|
||||
|
||||
def reference_loss_func(loss_sum_or_avg: torch.Tensor, num_measurements: torch.Tensor, take_avg_loss: bool):
|
||||
"""
|
||||
Returns average loss for data from``loss_sum_or_avg``. This function sums all losses from ``loss_sum_or_avg`` and
|
||||
divides the sum by the sum of ``num_measurements`` elements.
|
||||
|
||||
If ``take_avg_loss`` is ``True`` then ``loss_sum_or_avg[i]`` elements are mean values of ``num_measurements[i]``
|
||||
losses. In that case before computing sum of losses each element of ``loss_sum_or_avg`` is multiplied by
|
||||
corresponding element of ``num_measurements``.
|
||||
|
||||
If ``num_measurements`` sum is zero then the function returns NaN tensor.
|
||||
|
||||
The function is used for testing ``nemo.collections.common.metrics.GlobalAverageLossMetric`` class.
|
||||
|
||||
Args:
|
||||
loss_sum_or_avg: a one dimensional float ``torch.Tensor``. Sums or mean values of loss.
|
||||
num_measurements: a one dimensional integer ``torch.Tensor``. Number of values on which sums of means in
|
||||
``loss_sum_or_avg`` are calculated.
|
||||
take_avg_loss: if ``True`` then ``loss_sum_or_avg`` contains mean losses else ``loss_sum_or_avg`` contains
|
||||
sums of losses.
|
||||
"""
|
||||
if take_avg_loss:
|
||||
for i in range(loss_sum_or_avg.shape[0]):
|
||||
loss_sum_or_avg[i] *= num_measurements[i]
|
||||
nm_sum = num_measurements.sum()
|
||||
if nm_sum.eq(0):
|
||||
return torch.tensor(float('nan'))
|
||||
return loss_sum_or_avg.sum() / nm_sum
|
||||
|
||||
|
||||
def _loss_class_test(
|
||||
rank: int,
|
||||
worldsize: int,
|
||||
loss_sum_or_avg: Optional[torch.Tensor],
|
||||
num_measurements: Optional[torch.Tensor],
|
||||
dist_sync_on_step: bool,
|
||||
take_avg_loss: bool,
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
atol: float = 1e-8,
|
||||
):
|
||||
""" Utility function doing the actual comparison between lightning class metric
|
||||
and reference metric.
|
||||
Args:
|
||||
rank: rank of current process
|
||||
worldsize: number of processes
|
||||
loss_sum_or_avg: a one dimensional float torch tensor with loss sums or means.
|
||||
num_measurements: a one dimensional integer torch tensor with number of values on which sums or means from
|
||||
``loss_sum_or_avg`` were computed.
|
||||
dist_sync_on_step: bool, if true will synchronize metric state across
|
||||
processes at each ``forward()``
|
||||
take_avg_loss: dict with additional arguments used for class initialization
|
||||
check_dist_sync_on_step: bool, if true will check if the metric is also correctly
|
||||
calculated per batch per device (and not just at the end)
|
||||
check_batch: bool, if true will check if the metric is also correctly
|
||||
calculated across devices for each batch (and not just at the end)
|
||||
"""
|
||||
# Instantiate lightning metric
|
||||
loss_metric = GlobalAverageLossMetric(
|
||||
compute_on_step=True, dist_sync_on_step=dist_sync_on_step, take_avg_loss=take_avg_loss
|
||||
)
|
||||
|
||||
# verify perplexity works after being loaded from pickled state
|
||||
pickled_metric = pickle.dumps(loss_metric)
|
||||
loss_metric = pickle.loads(pickled_metric)
|
||||
|
||||
for i in range(rank, NUM_BATCHES, worldsize):
|
||||
batch_result = loss_metric(loss_sum_or_avg[i], num_measurements[i])
|
||||
if loss_metric.dist_sync_on_step:
|
||||
if rank == 0:
|
||||
ddp_loss_sum_or_avg = torch.stack([loss_sum_or_avg[i + r] for r in range(worldsize)])
|
||||
ddp_num_measurements = torch.stack([num_measurements[i + r] for r in range(worldsize)])
|
||||
sk_batch_result = reference_loss_func(ddp_loss_sum_or_avg, ddp_num_measurements, take_avg_loss)
|
||||
# assert for dist_sync_on_step
|
||||
if check_dist_sync_on_step:
|
||||
if sk_batch_result.isnan():
|
||||
assert batch_result.isnan()
|
||||
else:
|
||||
assert (
|
||||
np.allclose(batch_result.numpy(), sk_batch_result, atol=atol),
|
||||
f"batch_result = {batch_result.numpy()}, sk_batch_result = {sk_batch_result}",
|
||||
)
|
||||
else:
|
||||
ls = loss_sum_or_avg[i : i + 1]
|
||||
nm = num_measurements[i : i + 1]
|
||||
sk_batch_result = reference_loss_func(ls, nm, take_avg_loss)
|
||||
# assert for batch
|
||||
if check_batch:
|
||||
if sk_batch_result.isnan():
|
||||
assert batch_result.isnan()
|
||||
else:
|
||||
assert (
|
||||
np.allclose(batch_result.numpy(), sk_batch_result, atol=atol),
|
||||
f"batch_result = {batch_result.numpy()}, sk_batch_result = {sk_batch_result}",
|
||||
)
|
||||
# check on all batches on all ranks
|
||||
result = loss_metric.compute()
|
||||
assert isinstance(result, torch.Tensor)
|
||||
sk_result = reference_loss_func(loss_sum_or_avg, num_measurements, take_avg_loss)
|
||||
|
||||
# assert after aggregation
|
||||
if sk_result.isnan():
|
||||
assert result.isnan()
|
||||
else:
|
||||
assert (
|
||||
np.allclose(result.numpy(), sk_result, atol=atol),
|
||||
f"result = {result.numpy()}, sk_result = {sk_result}",
|
||||
)
|
||||
|
||||
|
||||
class LossTester(MetricTester):
|
||||
def run_class_loss_test(
|
||||
self,
|
||||
ddp: bool,
|
||||
loss_sum_or_avg: torch.Tensor,
|
||||
num_measurements: torch.Tensor,
|
||||
dist_sync_on_step: bool,
|
||||
take_avg_loss: bool,
|
||||
check_dist_sync_on_step: bool = True,
|
||||
check_batch: bool = True,
|
||||
):
|
||||
if ddp:
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("DDP not supported on windows")
|
||||
self.pool.starmap(
|
||||
partial(
|
||||
_loss_class_test,
|
||||
loss_sum_or_avg=loss_sum_or_avg,
|
||||
num_measurements=num_measurements,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
take_avg_loss=take_avg_loss,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
),
|
||||
[(rank, self.poolSize) for rank in range(self.poolSize)],
|
||||
)
|
||||
else:
|
||||
_loss_class_test(
|
||||
0,
|
||||
1,
|
||||
loss_sum_or_avg=loss_sum_or_avg,
|
||||
num_measurements=num_measurements,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
take_avg_loss=take_avg_loss,
|
||||
check_dist_sync_on_step=check_dist_sync_on_step,
|
||||
check_batch=check_batch,
|
||||
atol=self.atol,
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from .loss_inputs import ALL_NUM_MEASUREMENTS_ARE_ZERO, NO_ZERO_NUM_MEASUREMENTS, SOME_NUM_MEASUREMENTS_ARE_ZERO
|
||||
from .perplexity_inputs import NO_PROBS_NO_LOGITS, ONLY_LOGITS1, ONLY_LOGITS100, ONLY_PROBS, PROBS_AND_LOGITS
|
||||
from .pl_utils import LossTester, PerplexityTester
|
||||
from nemo.collections.common.metrics.classification_accuracy import TopKClassificationAccuracy
|
||||
|
||||
|
||||
class TestCommonMetrics:
|
||||
top_k_logits = torch.tensor([[0.1, 0.3, 0.2, 0.0], [0.9, 0.6, 0.2, 0.3], [0.2, 0.1, 0.4, 0.3]],) # 1 # 0 # 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_top_1_accuracy(self):
|
||||
labels = torch.tensor([0, 0, 2], dtype=torch.long)
|
||||
|
||||
accuracy = TopKClassificationAccuracy(top_k=None)
|
||||
acc = accuracy(logits=self.top_k_logits, labels=labels)
|
||||
|
||||
assert accuracy.correct_counts_k.shape == torch.Size([1])
|
||||
assert accuracy.total_counts_k.shape == torch.Size([1])
|
||||
assert abs(acc[0] - 0.667) < 1e-3
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_top_1_2_accuracy(self):
|
||||
labels = torch.tensor([0, 1, 0], dtype=torch.long)
|
||||
|
||||
accuracy = TopKClassificationAccuracy(top_k=[1, 2])
|
||||
top1_acc, top2_acc = accuracy(logits=self.top_k_logits, labels=labels)
|
||||
|
||||
assert accuracy.correct_counts_k.shape == torch.Size([2])
|
||||
assert accuracy.total_counts_k.shape == torch.Size([2])
|
||||
|
||||
assert abs(top1_acc - 0.0) < 1e-3
|
||||
assert abs(top2_acc - 0.333) < 1e-3
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_top_1_accuracy_distributed(self):
|
||||
# Simulate test on 2 process DDP execution
|
||||
labels = torch.tensor([[0, 0, 2], [2, 0, 0]], dtype=torch.long)
|
||||
|
||||
accuracy = TopKClassificationAccuracy(top_k=None)
|
||||
proc1_acc = accuracy(logits=self.top_k_logits, labels=labels[0])
|
||||
correct1, total1 = accuracy.correct_counts_k, accuracy.total_counts_k
|
||||
|
||||
accuracy.reset()
|
||||
proc2_acc = accuracy(logits=torch.flip(self.top_k_logits, dims=[1]), labels=labels[1]) # reverse logits
|
||||
correct2, total2 = accuracy.correct_counts_k, accuracy.total_counts_k
|
||||
|
||||
correct = torch.stack([correct1, correct2])
|
||||
total = torch.stack([total1, total2])
|
||||
|
||||
assert correct.shape == torch.Size([2, 1])
|
||||
assert total.shape == torch.Size([2, 1])
|
||||
|
||||
assert abs(proc1_acc[0] - 0.667) < 1e-3 # 2/3
|
||||
assert abs(proc2_acc[0] - 0.333) < 1e-3 # 1/3
|
||||
|
||||
accuracy.reset()
|
||||
accuracy.correct_counts_k = torch.tensor([correct.sum()])
|
||||
accuracy.total_counts_k = torch.tensor([total.sum()])
|
||||
acc_topk = accuracy.compute()
|
||||
acc_top1 = acc_topk[0]
|
||||
|
||||
assert abs(acc_top1 - 0.5) < 1e-3 # 3/6
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_top_1_accuracy_distributed_uneven_batch(self):
|
||||
# Simulate test on 2 process DDP execution
|
||||
accuracy = TopKClassificationAccuracy(top_k=None)
|
||||
|
||||
proc1_acc = accuracy(logits=self.top_k_logits, labels=torch.tensor([0, 0, 2]))
|
||||
correct1, total1 = accuracy.correct_counts_k, accuracy.total_counts_k
|
||||
|
||||
proc2_acc = accuracy(
|
||||
logits=torch.flip(self.top_k_logits, dims=[1])[:2, :], # reverse logits, select first 2 samples
|
||||
labels=torch.tensor([2, 0]),
|
||||
) # reduce number of labels
|
||||
correct2, total2 = accuracy.correct_counts_k, accuracy.total_counts_k
|
||||
|
||||
correct = torch.stack([correct1, correct2])
|
||||
total = torch.stack([total1, total2])
|
||||
|
||||
assert correct.shape == torch.Size([2, 1])
|
||||
assert total.shape == torch.Size([2, 1])
|
||||
|
||||
assert abs(proc1_acc[0] - 0.667) < 1e-3 # 2/3
|
||||
assert abs(proc2_acc[0] - 0.500) < 1e-3 # 1/2
|
||||
|
||||
accuracy.correct_counts_k = torch.tensor([correct.sum()])
|
||||
accuracy.total_counts_k = torch.tensor([total.sum()])
|
||||
acc_topk = accuracy.compute()
|
||||
acc_top1 = acc_topk[0]
|
||||
|
||||
assert abs(acc_top1 - 0.6) < 1e-3 # 3/5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ddp", [True, False])
|
||||
@pytest.mark.parametrize("dist_sync_on_step", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"probs, logits",
|
||||
[
|
||||
(ONLY_PROBS.probs, ONLY_PROBS.logits),
|
||||
(ONLY_LOGITS1.probs, ONLY_LOGITS1.logits),
|
||||
(ONLY_LOGITS100.probs, ONLY_LOGITS100.logits),
|
||||
(PROBS_AND_LOGITS.probs, PROBS_AND_LOGITS.logits),
|
||||
(NO_PROBS_NO_LOGITS.probs, NO_PROBS_NO_LOGITS.logits),
|
||||
],
|
||||
)
|
||||
class TestPerplexity(PerplexityTester):
|
||||
def test_perplexity(self, ddp, dist_sync_on_step, probs, logits):
|
||||
self.run_class_perplexity_test(
|
||||
ddp=ddp, probs=probs, logits=logits, dist_sync_on_step=dist_sync_on_step,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ddp", [True, False])
|
||||
@pytest.mark.parametrize("dist_sync_on_step", [True, False])
|
||||
@pytest.mark.parametrize("take_avg_loss", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"loss_sum_or_avg, num_measurements",
|
||||
[
|
||||
(NO_ZERO_NUM_MEASUREMENTS.loss_sum_or_avg, NO_ZERO_NUM_MEASUREMENTS.num_measurements),
|
||||
(SOME_NUM_MEASUREMENTS_ARE_ZERO.loss_sum_or_avg, SOME_NUM_MEASUREMENTS_ARE_ZERO.num_measurements),
|
||||
(ALL_NUM_MEASUREMENTS_ARE_ZERO.loss_sum_or_avg, ALL_NUM_MEASUREMENTS_ARE_ZERO.num_measurements),
|
||||
],
|
||||
)
|
||||
class TestLoss(LossTester):
|
||||
def test_loss(self, ddp, dist_sync_on_step, loss_sum_or_avg, num_measurements, take_avg_loss):
|
||||
self.run_class_loss_test(
|
||||
ddp=ddp,
|
||||
loss_sum_or_avg=loss_sum_or_avg,
|
||||
num_measurements=num_measurements,
|
||||
dist_sync_on_step=dist_sync_on_step,
|
||||
take_avg_loss=take_avg_loss,
|
||||
)
|
||||
@@ -0,0 +1,267 @@
|
||||
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pytest
|
||||
|
||||
from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer
|
||||
from nemo.collections.common.tokenizers.youtokentome_tokenizer import YouTokenToMeTokenizer
|
||||
|
||||
MODEL_SPECIAL_TOKENS = {
|
||||
'unk_token': '[UNK]',
|
||||
'sep_token': '[SEP]',
|
||||
'pad_token': '[PAD]',
|
||||
'bos_token': '[CLS]',
|
||||
'mask_token': '[MASK]',
|
||||
'eos_token': '[SEP]',
|
||||
'cls_token': '[CLS]',
|
||||
}
|
||||
|
||||
|
||||
class TestSentencePieceTokenizerLegacy:
|
||||
model_name = "/m_common.model"
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_add_special_tokens(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
assert tokenizer.vocab_size == tokenizer.original_vocab_size + len(set(special_tokens.values()))
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_tokens(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
|
||||
assert len(tokens) == len(text.split())
|
||||
assert tokens.count("[CLS]") == 1
|
||||
assert tokens.count("[MASK]") == 1
|
||||
assert tokens.count("[SEP]") == 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_text(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
result = tokenizer.tokens_to_text(tokens)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_ids(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
ids = tokenizer.text_to_ids(text)
|
||||
|
||||
assert len(ids) == len(text.split())
|
||||
assert ids.count(tokenizer.token_to_id("[CLS]")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("[MASK]")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("[SEP]")) == 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_text(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
ids = tokenizer.text_to_ids(text)
|
||||
result = tokenizer.ids_to_text(ids)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_ids(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
|
||||
assert len(ids) == len(tokens)
|
||||
assert ids.count(tokenizer.token_to_id("[CLS]")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("[MASK]")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("[SEP]")) == 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_tokens(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name, legacy=True)
|
||||
special_tokens = MODEL_SPECIAL_TOKENS
|
||||
tokenizer.add_special_tokens(special_tokens)
|
||||
|
||||
text = "[CLS] a b c [MASK] e f [SEP] g h i [SEP]"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
result = tokenizer.ids_to_tokens(ids)
|
||||
|
||||
assert len(result) == len(tokens)
|
||||
|
||||
for i in range(len(result)):
|
||||
assert result[i] == tokens[i]
|
||||
|
||||
|
||||
class TestSentencePieceTokenizer:
|
||||
model_name = "/m_new.model"
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_tokens(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
# <cls> is user_defined_symbol in the test tokenizer model
|
||||
# <unk>, <sep>, <s>, and </s> are control symbols
|
||||
text = "<cls> a b c <sep> e f g h i </s>"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
|
||||
assert tokens.count("<cls>") == 1
|
||||
assert tokens.count("<sep>") == 0
|
||||
assert tokens.count("</s>") == 0
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_text(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
# <cls> is user_defined_symbol in the test tokenizer model
|
||||
text = "<cls> a b c e f g h i"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
result = tokenizer.tokens_to_text(tokens)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_ids(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
# <cls> is user_defined_symbol in the test tokenizer model
|
||||
# <unk>, <sep>, <s>, and </s> are control symbols
|
||||
text = "<cls> a b c <sep> e f g h i </s>"
|
||||
tokens = tokenizer.text_to_ids(text)
|
||||
|
||||
assert tokens.count(tokenizer.token_to_id("<cls>")) == 1
|
||||
assert tokens.count(tokenizer.token_to_id("<sep>")) == 0
|
||||
assert tokens.count(tokenizer.token_to_id("</s>")) == 0
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_text(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
text = "<cls> a b c <sep> e f g h i </s>"
|
||||
ids = tokenizer.text_to_ids(text)
|
||||
result = tokenizer.ids_to_text(ids)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_ids(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
tokens = ["<cls>", "a", "b", "c", "<sep>", "e", "f", "<sep>", "g", "h", "i", "</s>"]
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
|
||||
assert len(ids) == len(tokens)
|
||||
assert ids.count(tokenizer.token_to_id("<cls>")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("</s>")) == 1
|
||||
assert ids.count(tokenizer.token_to_id("<sep>")) == 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_tokens(self, test_data_dir):
|
||||
tokenizer = SentencePieceTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
tokens = ["<cls>", "a", "b", "c", "<sep>", "e", "f", "<sep>", "g", "h", "i", "</s>"]
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
result = tokenizer.ids_to_tokens(ids)
|
||||
|
||||
assert len(result) == len(tokens)
|
||||
|
||||
for i in range(len(result)):
|
||||
assert result[i] == tokens[i]
|
||||
|
||||
|
||||
class TestYouTokenToMeTokenizer:
|
||||
model_name = "/yttm.4096.en-de.model"
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_tokens(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
text = "<BOS> a b c e <UNK> f g h i <EOS>"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
|
||||
assert tokens.count("<BOS>") == 0
|
||||
assert tokens.count("<UNK>") == 0
|
||||
assert tokens.count("<EOS>") == 0
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_text(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
text = "a b c e f g h i"
|
||||
tokens = tokenizer.text_to_tokens(text)
|
||||
result = tokenizer.tokens_to_text(tokens)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_text_to_ids(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
text = "<BOS> a b c <UNK> e f g h i <EOS>"
|
||||
tokens = tokenizer.text_to_ids(text)
|
||||
|
||||
assert tokens.count(tokenizer.bos_id) == 0
|
||||
assert tokens.count(tokenizer.unk_id) == 0
|
||||
assert tokens.count(tokenizer.eos_id) == 0
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_text(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
text = "a b c e f g h i"
|
||||
ids = tokenizer.text_to_ids(text)
|
||||
result = tokenizer.ids_to_text(ids)
|
||||
|
||||
assert text == result
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tokens_to_ids(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
tokens = ["<BOS>", "a", "b", "c", "<UNK>", "e", "f", "<UNK>", "g", "h", "i", "<EOS>"]
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
|
||||
assert len(ids) == len(tokens)
|
||||
assert ids.count(tokenizer.bos_id) == 1
|
||||
assert ids.count(tokenizer.eos_id) == 1
|
||||
assert ids.count(tokenizer.unk_id) == 2
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ids_to_tokens(self, test_data_dir):
|
||||
tokenizer = YouTokenToMeTokenizer(test_data_dir + self.model_name)
|
||||
|
||||
tokens = ["<BOS>", "a", "b", "c", "<UNK>", "e", "f", "<UNK>", "g", "h", "i", "<EOS>"]
|
||||
ids = tokenizer.tokens_to_ids(tokens)
|
||||
result = tokenizer.ids_to_tokens(ids)
|
||||
|
||||
assert len(result) == len(tokens)
|
||||
|
||||
for i in range(len(result)):
|
||||
assert result[i] == tokens[i]
|
||||
Reference in New Issue
Block a user