# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import csv import os import re import string from typing import Dict, List, Union __all__ = [ "read_wordids", "set_grapheme_case", "GRAPHEME_CASE_UPPER", "GRAPHEME_CASE_LOWER", "GRAPHEME_CASE_MIXED", "get_heteronym_spans", ] # define grapheme cases. GRAPHEME_CASE_UPPER = "upper" GRAPHEME_CASE_LOWER = "lower" GRAPHEME_CASE_MIXED = "mixed" def read_wordids(wordid_map: str): """ Reads wordid file from WikiHomograph dataset, https://github.com/google-research-datasets/WikipediaHomographData/blob/master/data/wordids.tsv Args: wordid_map: path to wordids.tsv Returns: data_dict: a dictionary of graphemes with corresponding word_id - ipa_form pairs wordid_to_idx: word id to label id mapping """ if not os.path.exists(wordid_map): raise ValueError(f"{wordid_map} not found") data_dict = {} wordid_to_idx = {} with open(wordid_map, "r", encoding="utf-8") as f: tsv_file = csv.reader(f, delimiter="\t") for i, line in enumerate(tsv_file): if i == 0: continue grapheme = line[0] word_id = line[1] ipa_form = line[3] wordid_to_idx[word_id] = len(wordid_to_idx) if grapheme not in data_dict: data_dict[grapheme] = {} data_dict[grapheme][word_id] = ipa_form return data_dict, wordid_to_idx def get_wordid_to_phonemes(wordid_to_phonemes_file: str, to_lower: bool = True): """ WikiHomograph and NeMo use slightly different phoneme sets, this function reads WikiHomograph word_ids to NeMo IPA heteronyms mapping. Args: wordid_to_phonemes_file: Path to a file with mapping from wordid predicted by the model to phonemes, e.g., NeMo/scripts/tts_dataset_files/wordid_to_ipa-0.7b_nv22.10.tsv to_lower: set to True to lower case wordid """ if not os.path.exists(wordid_to_phonemes_file): raise ValueError(f"{wordid_to_phonemes_file} not found") wordid_to_nemo_cmu = {} with open(wordid_to_phonemes_file, "r", encoding="utf-8") as f: for i, line in enumerate(f): if to_lower: line = line.lower() line = line.strip().split(" ") wordid_to_nemo_cmu[line[0]] = line[1] return wordid_to_nemo_cmu def remove_punctuation(text: str, remove_spaces: bool = False, do_lower: bool = False, exclude: List[str] = None): """ Remove punctuation marks form text Args: text: input text remove_spaces: set to True to remove spaces do_lower: set to True to lower case the text exclude: specify list of punctuation marks keep in the output, e.g., exclude=["'", "."] Return: processed text with punctuation marks removed """ all_punct_marks = string.punctuation if exclude is not None: for p in exclude: all_punct_marks = all_punct_marks.replace(p, "") text = re.sub("[" + all_punct_marks + "]", " ", text) text = re.sub(r" +", " ", text) if remove_spaces: text = text.replace(" ", "").replace("\u00A0", "").strip() if do_lower: text = text.lower() return text.strip() def get_heteronym_spans(sentences: List[str], supported_heteronyms: Union[Dict, List]): """ Find heteronyms in sentences and returns span indices Args: sentences: sentences to find heteronyms in supported_heteronyms: heteronyms to look for Return: start_end: List[Tuple[int]] - start-end indices that indicate location of found heteronym in the sentence heteronyms: List[List[str]] - heteronyms found in sentences, each sentence can contain more than one heteronym """ start_end = [] heteronyms = [] for sent in sentences: cur_start_end = [] cur_heteronyms = [] start_idx = 0 for word in sent.lower().split(): word_by_hyphen = word.split("-") for sub_word in word_by_hyphen: no_punct_word = remove_punctuation(sub_word, do_lower=True, remove_spaces=False) if no_punct_word in supported_heteronyms: start_idx = sent.lower().index(no_punct_word, start_idx) end_idx = start_idx + len(no_punct_word) cur_start_end.append((start_idx, end_idx)) cur_heteronyms.append(no_punct_word) start_idx = end_idx else: start_idx += len(sub_word) + 1 heteronyms.append(cur_heteronyms) start_end.append(cur_start_end) return start_end, heteronyms def set_grapheme_case(text: str, case: str = "upper") -> str: if case == "upper": text_new = text.upper() elif case == "lower": text_new = text.lower() elif case == "mixed": # keep as-is, mix-cases text_new = text else: raise ValueError(f"Case <{case}> is not supported. Please specify either 'upper', 'lower', or 'mixed'.") return text_new