mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-19 07:54:56 +00:00
Brandon Thomas
121 lines
4.0 KiB
Python
121 lines
4.0 KiB
Python
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
from os.path import expanduser
|
|
from typing import Any, Callable, Dict, Iterator, List, Optional, Union
|
|
|
|
|
|
class ManifestBase:
|
|
def __init__(self, *args, **kwargs):
|
|
raise ValueError(
|
|
"This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour."
|
|
)
|
|
|
|
|
|
class ManifestEN:
|
|
def __init__(self, *args, **kwargs):
|
|
raise ValueError(
|
|
"This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour."
|
|
)
|
|
|
|
|
|
def item_iter(
|
|
manifests_files: Union[str, List[str]], parse_func: Callable[[str, Optional[str]], Dict[str, Any]] = None
|
|
) -> Iterator[Dict[str, Any]]:
|
|
"""Iterate through json lines of provided manifests.
|
|
|
|
NeMo ASR pipelines often assume certain manifest files structure. In
|
|
particular, each manifest file should consist of line-per-sample files with
|
|
each line being correct json dict. Each such json dict should have a field
|
|
for audio file string, a field for duration float and a field for text
|
|
string. Offset also could be additional field and is set to None by
|
|
default.
|
|
|
|
Args:
|
|
manifests_files: Either single string file or list of such -
|
|
manifests to yield items from.
|
|
|
|
parse_func: A callable function which accepts as input a single line
|
|
of a manifest and optionally the manifest file itself,
|
|
and parses it, returning a dictionary mapping from str -> Any.
|
|
|
|
Yields:
|
|
Parsed key to value item dicts.
|
|
|
|
Raises:
|
|
ValueError: If met invalid json line structure.
|
|
"""
|
|
|
|
if isinstance(manifests_files, str):
|
|
manifests_files = [manifests_files]
|
|
|
|
if parse_func is None:
|
|
parse_func = __parse_item
|
|
|
|
k = -1
|
|
for manifest_file in manifests_files:
|
|
with open(expanduser(manifest_file), 'r') as f:
|
|
for line in f:
|
|
k += 1
|
|
item = parse_func(line, manifest_file)
|
|
item['id'] = k
|
|
|
|
yield item
|
|
|
|
|
|
def __parse_item(line: str, manifest_file: str) -> Dict[str, Any]:
|
|
item = json.loads(line)
|
|
|
|
# Audio file
|
|
if 'audio_filename' in item:
|
|
item['audio_file'] = item.pop('audio_filename')
|
|
elif 'audio_filepath' in item:
|
|
item['audio_file'] = item.pop('audio_filepath')
|
|
else:
|
|
raise ValueError(
|
|
f"Manifest file {manifest_file} has invalid json line structure: {line} without proper audio file key."
|
|
)
|
|
item['audio_file'] = expanduser(item['audio_file'])
|
|
|
|
# Duration.
|
|
if 'duration' not in item:
|
|
raise ValueError(
|
|
f"Manifest file {manifest_file} has invalid json line structure: {line} without proper duration key."
|
|
)
|
|
|
|
# Text.
|
|
if 'text' in item:
|
|
pass
|
|
elif 'text_filepath' in item:
|
|
with open(item.pop('text_filepath'), 'r') as f:
|
|
item['text'] = f.read().replace('\n', '')
|
|
elif 'normalized_text' in item:
|
|
item['text'] = item['normalized_text']
|
|
else:
|
|
raise ValueError(
|
|
f"Manifest file {manifest_file} has invalid json line structure: {line} without proper text key."
|
|
)
|
|
|
|
item = dict(
|
|
audio_file=item['audio_file'],
|
|
duration=item['duration'],
|
|
text=item['text'],
|
|
offset=item.get('offset', None),
|
|
speaker=item.get('speaker', None),
|
|
orig_sr=item.get('orig_sample_rate', None),
|
|
)
|
|
|
|
return item
|