






"""simple docstring"""




import json
import os
import unittest
from typing import Tuple

from transformers import WavaVecaPhonemeCTCTokenizer
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.models.wavaveca_phoneme.tokenization_wavaveca_phoneme import WavaVecaPhonemeCTCTokenizerOutput
from transformers.testing_utils import require_phonemizer

from ...test_tokenization_common import TokenizerTesterMixin



@require_phonemizer
class   lowerCamelCase				(_snake_case			,  unittest.TestCase      ):







		'''simple docstring'''



		_snake_case   :   List[str]							=  WavaVecaPhonemeCTCTokenizer
		_snake_case   :   Dict							=  False



		def  __UpperCAmelCase      (							self       )     ->							str:
							super().setUp()

							UpperCAmelCase_   :				List[str]						=						(
							    '<s> <pad> </s> <unk> n s t ə l a i k d m ɛ ɾ e ɪ p o ɐ z ð f j v b ɹ ʁ ʊ iː r w ʌ u ɡ æ aɪ ʃ h ɔ ɑː '
							    'ŋ ɚ eɪ β uː y ɑ̃ oʊ ᵻ eː θ aʊ ts oː ɔ̃ ɣ ɜ ɑ dʒ əl x ɜː ç ʒ tʃ ɔː ɑːɹ ɛ̃ ʎ ɔːɹ ʋ aː ɕ œ ø oːɹ ɲ yː '
							    'ʔ iə i5 s. tɕ ?? nʲ ɛː œ̃ ɭ ɔø ʑ tʲ ɨ ɛɹ ts. rʲ ɪɹ ɭʲ i.5 ɔɪ q sʲ u5 ʊɹ iɜ a5 iɛ5 øː ʕ ja əɜ th ɑ5 '
							    'oɪ dʲ ə5 tɕh ts.h mʲ ɯ dʑ vʲ e̞ tʃʲ ei5 o5 onɡ5 ɑu5 iɑ5 ai5 aɪɚ kh ə1 ʐ i2 ʉ ħ t[ aɪə ʲ ju ə2 u2 oɜ '
							    'pː iɛɜ ou5 y5 uɜ tː uo5 d[ uoɜ tsh ɑɜ ɵ i̪5 uei5 ɟ aɜ ɑɨ i.ɜ eʊ o2 ɐ̃ ä pʲ kʲ n̩ ɒ ph ɑu2 uɨ əɪ ɫ ɬ '
							    'yɜ bʲ ɑ2 s̪ aiɜ χ ɐ̃ʊ̃ 1 ə4 yæɜ a2 ɨː t̪ iouɜ ũ onɡɜ aɨ iɛ2 ɔɨ ɑuɜ o̞ ei2 iou2 c kː y2 ɖ oe dˤ yɛɜ '
							    'əʊ S ɡʲ onɡ2 u" eiɜ ʈ ɯᵝ iou5 dZ r̝̊ i.2 tS s^ ʝ yə5 iɑɜ uə5 pf ɨu iɑ2 ou2 ər2 fʲ ai2 r̝ uəɜ ɳ əɨ '
							    'ua5 uɪ ɽ bː yu5 uo2 yɛ5 l̩ ɻ ərɜ ʂ i̪2 ouɜ uaɜ a. a.ː yæ5 dː r̩ ee ɪu ər5 i̪ ɜ æi u: i.ː t^ o1 ɪ^ '
							    'ai ueiɜ æː ɛɪ eə i. ɴ ie ua2 ɑ1 o4 tʃː o: ɑ: u1 N i̪1 au yæ2 u. qː yəɜ y: kʰ tʃʰ iʊ sx õ uo tʰ '
							    'uai5 bʰ u.ː uə2 ʊə d^ s̪ː yiɜ dʰ r. oe: i1 ɟː yu2 nʲʲ i̪4 uei2 tsʲ ɸ ĩ ɑ4 t̪ː eɑ u4 e: tsː ʈʰ ɡʰ '
							    'ɯɯ dʒʲ ʂʲ X ɵː uaiɜ tɕʲ ã t^ː ẽː yɛ2 cː i.1 ɛʊ dˤdˤ dʒː i4 ɡː yi ɕʲ ɟʰ pʰ dʑʲ yuɜ ua1 ua4 æiː ɐɐ '
							    'ui iou1 ʊː a1 iou4 cʰ iɛ1 yə2 ɖʰ ẽ ʒʲ ää ər4 iːː ɪː iɑ1 ər1 œː øi ɪuː cʰcʰ əː1 iː1 ũ kʰː o̞o̞ xʲ '
							    'ou1 iɛ4 e̞e̞ y1 dzː dʲʲ dʰː ɯᵝɯᵝ lː uo1 i.4 i: yɛ5ʲ a4'
							).split(' '       )
							UpperCAmelCase_   :				Optional[int]						=						dict(zip(_UpperCamelCase    ,		range(len(_UpperCamelCase       )       )       )       )

							UpperCAmelCase_   :				int						=						{'pad_token': '<pad>', 'unk_token': '<unk>', 'bos_token': '<s>', 'eos_token': '</s>'}

							UpperCAmelCase_   :				Optional[Any]						=						os.path.join(self.tmpdirname    ,		VOCAB_FILES_NAMES['vocab_file']       )
							with open(self.vocab_file    ,		'w'    ,		encoding='utf-8'       ) as fp:
												fp.write(json.dumps(_UpperCamelCase       ) + '\n'       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase=False    ,		_UpperCamelCase=2_0    ,		_UpperCamelCase=5       )     ->							Tuple[str, list]:
							UpperCAmelCase_   :				Dict						=						[(i, tokenizer.decode([i]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )) for i in range(len(_UpperCamelCase       )       )]
							UpperCAmelCase_   :				str						=						list(filter(lambda _UpperCamelCase       : [t[0]] == tokenizer.encode(t[1]    ,		do_phonemize=_UpperCamelCase       )    ,		_UpperCamelCase       )       )
							if max_length is not None and len(_UpperCamelCase       ) > max_length:
												UpperCAmelCase_   :				List[str]						=						toks[:max_length]
							if min_length is not None and len(_UpperCamelCase       ) < min_length and len(_UpperCamelCase       ) > 0:
												while len(_UpperCamelCase       ) < min_length:
																	UpperCAmelCase_   :				Any						=						toks + toks
        # toks_str = [t[1] for t in toks]
							UpperCAmelCase_   :				Optional[Any]						=						[t[0] for t in toks]

							# Ensure consistency
							UpperCAmelCase_   :				Optional[Any]						=						tokenizer.decode(_UpperCamelCase    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
							if " " not in output_txt and len(_UpperCamelCase       ) > 1:
												UpperCAmelCase_   :				Any						=						(
												    tokenizer.decode([toks_ids[0]]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
												    + ' '
												    + tokenizer.decode(toks_ids[1:]    ,		clean_up_tokenization_spaces=_UpperCamelCase       )
												)
							if with_prefix_space:
												UpperCAmelCase_   :				Tuple						=						' ' + output_txt
							UpperCAmelCase_   :				Optional[Any]						=						tokenizer.encode(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase       )
							return output_txt, output_ids



		def  __UpperCAmelCase      (							self    ,		**_UpperCamelCase       )     ->							List[str]:
							kwargs.update(self.special_tokens_map       )
							return WavaVecaPhonemeCTCTokenizer.from_pretrained(self.tmpdirname    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				Any						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )

							# check adding a single token
							tokenizer.add_tokens('xxx'       )
							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer('m xxx ɪ'    ,		do_phonemize=_UpperCamelCase       ).input_ids
							self.assertEqual(_UpperCamelCase    ,		[1_3, 3_9_2, 1_7]       )  # xxx should be last token

							tokenizer.add_tokens(['aaa', 'bbb', 'ccc']       )
							UpperCAmelCase_   :				int						=						tokenizer('m aaa ɪ ccc'    ,		do_phonemize=_UpperCamelCase       ).input_ids
							self.assertEqual(_UpperCamelCase    ,		[1_3, 3_9_3, 1_7, 3_9_5]       )  # aaa and ccc should be after xxx and 2 after aaa

							UpperCAmelCase_   :				List[str]						=						tokenizer('maɪ c'    ,		do_phonemize=_UpperCamelCase       ).input_ids
							self.assertEqual(_UpperCamelCase    ,		[3, 2_0_0]       )  # mai should be <unk> (=3)



		def  __UpperCAmelCase      (							self       )     ->							Dict:
							UpperCAmelCase_   :				Dict						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )

							UpperCAmelCase_   :				Tuple						=						'Hello how are you'
							UpperCAmelCase_   :				Any						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )
							self.assertEqual(_UpperCamelCase    ,		'h ə l oʊ h aʊ ɑːɹ j uː'       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[int]:
							UpperCAmelCase_   :				Any						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )

							UpperCAmelCase_   :				Optional[int]						=						'Hello how are you'
							UpperCAmelCase_   :				Dict						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )
							self.assertEqual(tokenizer(_UpperCamelCase       ).input_ids    ,		tokenizer(_UpperCamelCase    ,		do_phonemize=_UpperCamelCase       ).input_ids       )



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							UpperCAmelCase_   :				List[Any]						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )
							UpperCAmelCase_   :				int						=						'Hello how are you'
							UpperCAmelCase_   :				str						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )

							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer.decode(tokenizer(_UpperCamelCase       ).input_ids       )

							self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							UpperCAmelCase_   :				Optional[int]						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )

							UpperCAmelCase_   :				List[str]						=						[
							    [1_1, 5, 1_5, tokenizer.pad_token_id, 1_5, 8, 9_8],
							    [2_4, 2_2, 5, 2_4, 2_2, 5, 7_7],
							]
							UpperCAmelCase_   :				List[Any]						=						tokenizer.decode(sample_ids[0]       )
							UpperCAmelCase_   :				Optional[int]						=						tokenizer.batch_decode(_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		batch_tokens[0]       )
							self.assertEqual(_UpperCamelCase    ,		['k s ɾ ɾ l ɭʲ', 'j ð s j ð s oːɹ']       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_   :				int						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							UpperCAmelCase_   :				Any						=						'Hello how are you'
							UpperCAmelCase_   :				int						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )
							self.assertEqual(_UpperCamelCase    ,		'h ə l oʊ | h aʊ | ɑːɹ | j uː |'       )



		def  __UpperCAmelCase      (							self       )     ->							Dict:
							UpperCAmelCase_   :				Dict						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							UpperCAmelCase_   :				Dict						=						'Hello how are you'
							UpperCAmelCase_   :				Dict						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )
							self.assertEqual(tokenizer(_UpperCamelCase       ).input_ids    ,		tokenizer(_UpperCamelCase    ,		do_phonemize=_UpperCamelCase       ).input_ids       )



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							UpperCAmelCase_   :				Optional[Any]						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							# fmt: off
							UpperCAmelCase_   :				Any						=						[
							    [1_1, 5, 1_5, tokenizer.pad_token_id, tokenizer.word_delimiter_token_id, 1_5, 8, tokenizer.word_delimiter_token_id, 9_8],
							    [tokenizer.word_delimiter_token_id, 2_4, 2_2, tokenizer.word_delimiter_token_id, 5, 2_4, 2_2, 5, 7_7],
							]
							# fmt: on

							# decode with word_del_token filter
							UpperCAmelCase_   :				List[str]						=						tokenizer.decode(sample_ids[0]       )
							UpperCAmelCase_   :				Dict						=						tokenizer.batch_decode(_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		batch_tokens[0]       )
							self.assertEqual(_UpperCamelCase    ,		['k s ɾ ɾ l ɭʲ', 'j ð s j ð s oːɹ']       )

							# decode with no word_del_token filter
							UpperCAmelCase_   :				List[Any]						=						tokenizer.decode(sample_ids[0]    ,		filter_word_delimiter_token=_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						tokenizer.batch_decode(_UpperCamelCase    ,		filter_word_delimiter_token=_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		batch_tokens[0]       )
							self.assertEqual(_UpperCamelCase    ,		['k s ɾ | ɾ l | ɭʲ', '| j ð | s j ð s oːɹ']       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				Tuple						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							UpperCAmelCase_   :				Any						=						'Hello how are you'
							UpperCAmelCase_   :				Optional[int]						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )

							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer.decode(tokenizer(_UpperCamelCase       ).input_ids    ,		filter_word_delimiter_token=_UpperCamelCase       )

							self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							UpperCAmelCase_   :				Tuple						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							UpperCAmelCase_   :				int						=						'Hello how are you'
							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer.phonemize(_UpperCamelCase    ,		phonemizer_lang='en-us'       )

							UpperCAmelCase_   :				int						=						tokenizer.decode(tokenizer(_UpperCamelCase       ).input_ids    ,		filter_word_delimiter_token=_UpperCamelCase       )

							self.assertEqual(' '.join([p.strip() for p in phonemes.split(' |'       )]       ).strip()    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				List[str]						=						self.tokenizer_class.from_pretrained(
							    'facebook/wav2vec2-lv-60-espeak-cv-ft'    ,		word_delimiter_token=_UpperCamelCase       )
							UpperCAmelCase_   :				Optional[int]						=						'Hello how are you'

							UpperCAmelCase_   :				Any						=						tokenizer(_UpperCamelCase    ,		phonemizer_lang='en-us'       ).input_ids
							UpperCAmelCase_   :				Optional[int]						=						tokenizer(_UpperCamelCase    ,		phonemizer_lang='fr-fr'       ).input_ids

							self.assertNotEqual(_UpperCamelCase    ,		_UpperCamelCase       )

							UpperCAmelCase_   :				Union[str, Any]						=						tokenizer.decode(_UpperCamelCase       )
							UpperCAmelCase_   :				Optional[Any]						=						tokenizer.decode(_UpperCamelCase       )

							self.assertEqual(_UpperCamelCase    ,		'h ə l oʊ h aʊ ɑːɹ j uː'       )
							self.assertEqual(_UpperCamelCase    ,		'ɛ l o h aʊ a ʁ j u'       )



		def  __UpperCAmelCase      (							self       )     ->							Any:
							UpperCAmelCase_   :				Union[str, Any]						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )
							UpperCAmelCase_   :				Any						=						'Hello how Are you'
							UpperCAmelCase_   :				Union[str, Any]						=						'hello how are you'

							UpperCAmelCase_   :				str						=						tokenizer(_UpperCamelCase       ).input_ids
							UpperCAmelCase_   :				List[str]						=						tokenizer(_UpperCamelCase       ).input_ids

							self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							UpperCAmelCase_   :				List[str]						=						self.tokenizer_class.from_pretrained('facebook/wav2vec2-lv-60-espeak-cv-ft'       )
							tokenizer.add_tokens(['!', '?']       )
							tokenizer.add_special_tokens({'cls_token': '$$$'}       )

							# fmt: off
							UpperCAmelCase_   :				Any						=						[
							    [1_1, 5, 1_5, tokenizer.pad_token_id, 1_5, 8, 9_8, 3_9_2, 3_9_2, 3_9_3, 3_9_2, 3_9_2, 3_9_3, 3_9_4, 3_9_4],
							    [2_4, 2_2, 5, 2_4, 2_2, 5, 7_7, tokenizer.pad_token_id, 3_9_4, 3_9_4],
							]
							# fmt: on

							UpperCAmelCase_   :				Optional[int]						=						tokenizer.batch_decode(_UpperCamelCase       )
							self.assertEqual(_UpperCamelCase    ,		['k s ɾ ɾ l ɭʲ!?!? $$$', 'j ð s j ð s oːɹ $$$']       )



		@staticmethod
		def  __UpperCAmelCase      (							_UpperCamelCase    ,		_UpperCamelCase       )     ->							List[Any]:
							UpperCAmelCase_   :				int						=						[d[key] for d in offsets]
							return retrieved_list



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							UpperCAmelCase_   :				Optional[int]						=						self.get_tokenizer(word_delimiter_token='|'       )
							tokenizer.add_tokens('|'       )

							# fmt: off
							# ksssɾɾ|ɾɾ<pad>ɾɾ|<pad>ɾlll|ɭʲ -> k s ɾ ɾ | ɾ l | ɭʲ"
							UpperCAmelCase_   :				Optional[int]						=						[1_1, 5, 5, 5, 1_5, 1_5, tokenizer.pad_token_id, 1_5, 1_5, tokenizer.word_delimiter_token_id, tokenizer.pad_token_id, 1_5, 8, 8, 8, tokenizer.word_delimiter_token_id, 9_8]
							# fmt: on

							UpperCAmelCase_   :				Optional[int]						=						tokenizer.decode(_UpperCamelCase    ,		output_char_offsets=_UpperCamelCase    ,		filter_word_delimiter_token=_UpperCamelCase       )
							# check Wav2Vec2CTCTokenizerOutput keys for char
							self.assertEqual(len(outputs.keys()       )    ,		2       )
							self.assertTrue('text' in outputs       )
							self.assertTrue('char_offsets' in outputs       )
							self.assertTrue(isinstance(_UpperCamelCase    ,		_UpperCamelCase       )       )

							# check that order of chars is correct and identical for both outputs
							self.assertEqual(' '.join(self.get_from_offsets(outputs['char_offsets']    ,		'char'       )       )    ,		outputs.text       )
							self.assertListEqual(
							    self.get_from_offsets(outputs['char_offsets']    ,		'char'       )    ,		['k', 's', 'ɾ', 'ɾ', '|', 'ɾ', 'l', '|', 'ɭʲ']       )

							# check that offsets are actually correct for char
							# 0-1 is 11, 1-4 is 5, 4-6 is first 15, 6-7 is <pad> (thus not shown), 7-9 is second 15, 9-10 is word_delimiter_token,
							# 10-11 is <pad> (thus not shown), 11-12 is third 15, 12-15 is 8, 15-16 is word_delimiter_token, 16-17 is 98
							self.assertListEqual(
							    self.get_from_offsets(outputs['char_offsets']    ,		'start_offset'       )    ,		[0, 1, 4, 7, 9, 1_1, 1_2, 1_5, 1_6]       )
							self.assertListEqual(
							    self.get_from_offsets(outputs['char_offsets']    ,		'end_offset'       )    ,		[1, 4, 6, 9, 1_0, 1_2, 1_5, 1_6, 1_7]       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[int]:
							UpperCAmelCase_   :				Optional[Any]						=						self.get_tokenizer(word_delimiter_token='|'       )

							def check_list_tuples_equal(_UpperCamelCase    ,		_UpperCamelCase       ):
												self.assertTrue(isinstance(_UpperCamelCase    ,		_UpperCamelCase       )       )
												self.assertTrue(isinstance(outputs_list[0]    ,		_UpperCamelCase       )       )

												# transform list to ModelOutput
												UpperCAmelCase_   :				Optional[int]						=						WavaVecaPhonemeCTCTokenizerOutput(
												    {k: [d[k] for d in outputs_list] for k in outputs_list[0]}       )

												self.assertListEqual(outputs_batch['text']    ,		outputs_batch_a['text']       )

												def recursive_check(_UpperCamelCase    ,		_UpperCamelCase       ):
																	if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
																						[recursive_check(_UpperCamelCase    ,		_UpperCamelCase       ) for la, la in zip(_UpperCamelCase    ,		_UpperCamelCase       )]
																	self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )

												if "char_offsets" in outputs_batch:
																	recursive_check(outputs_batch['char_offsets']    ,		outputs_batch_a['char_offsets']       )

        # fmt: off
							UpperCAmelCase_   :				int						=						[
							    [1_1, 5, 1_5, tokenizer.pad_token_id, 1_5, 4, 8, 9_8, 3_2, 3_2, 3_2, 3_2, 4, 3_3, tokenizer.word_delimiter_token_id, 3_2, 3_2, 3_3, 3_4, 3_4],
							    [2_4, 2_2, 5, tokenizer.word_delimiter_token_id, tokenizer.word_delimiter_token_id, 2_4, 2_2, 2_2, 2_2, 4, 5, 7_7, tokenizer.pad_token_id, 2_2, 2_2, 4, 3_4, 3_4, 3_4, 3_4],
							]
							# fmt: on

							# We assume that `decode` works as expected. All we will check now is
							# the output type is correct and the output is identical to `decode`

							# char
							UpperCAmelCase_   :				int						=						tokenizer.batch_decode(_UpperCamelCase    ,		output_char_offsets=_UpperCamelCase       )
							UpperCAmelCase_   :				List[Any]						=						[tokenizer.decode(_UpperCamelCase    ,		output_char_offsets=_UpperCamelCase       ) for ids in sample_ids]
							check_list_tuples_equal(_UpperCamelCase    ,		_UpperCamelCase       )



		@unittest.skip('Wav2Vec2PhonemeTokenizer always lower cases letters to correctly map to phonemes'       )
		def  __UpperCAmelCase      (							self       )     ->							str:
							pass



		@unittest.skip('Wav2Vec2PhonemeTokenizer always puts spaces between phonemes'       )
		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							pass



		@unittest.skip('encodes to text to ids, but decodes ids to phonemes -> not possible to have internal consistency'       )
		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							pass



		@unittest.skip('Wav2Vec2PhonemeModel has no max model length => no testing'       )
		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							pass



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							UpperCAmelCase_   :				Tuple						=						self.get_tokenizers(do_lower_case=_UpperCamelCase       )
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	UpperCAmelCase_   :				Tuple						=						tokenizer.vocab_size
																	UpperCAmelCase_   :				Dict						=						len(_UpperCamelCase       )

																	self.assertNotEqual(_UpperCamelCase    ,		0       )

																	# We usually have added tokens from the start in tests because our vocab fixtures are
																	# smaller than the original vocabs - let's not assert this
																	# self.assertEqual(vocab_size, all_size)

																	UpperCAmelCase_   :				Optional[Any]						=						['aaaaa bbbbbb', 'cccccccccdddddddd']
																	UpperCAmelCase_   :				Optional[Any]						=						tokenizer.add_tokens(_UpperCamelCase       )
																	UpperCAmelCase_   :				str						=						tokenizer.vocab_size
																	UpperCAmelCase_   :				Optional[int]						=						len(_UpperCamelCase       )

																	self.assertNotEqual(_UpperCamelCase    ,		0       )
																	self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )
																	self.assertEqual(_UpperCamelCase    ,		len(_UpperCamelCase       )       )
																	self.assertEqual(_UpperCamelCase    ,		all_size + len(_UpperCamelCase       )       )

																	UpperCAmelCase_   :				str						=						tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'    ,		add_special_tokens=_UpperCamelCase       )

																	self.assertGreaterEqual(len(_UpperCamelCase       )    ,		4       )
																	self.assertGreater(tokens[0]    ,		tokenizer.vocab_size - 1       )
																	self.assertGreater(tokens[-3]    ,		tokenizer.vocab_size - 1       )

																	UpperCAmelCase_   :				Tuple						=						{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
																	UpperCAmelCase_   :				Any						=						tokenizer.add_special_tokens(_UpperCamelCase       )
																	UpperCAmelCase_   :				List[str]						=						tokenizer.vocab_size
																	UpperCAmelCase_   :				Optional[Any]						=						len(_UpperCamelCase       )

																	self.assertNotEqual(_UpperCamelCase    ,		0       )
																	self.assertEqual(_UpperCamelCase    ,		_UpperCamelCase       )
																	self.assertEqual(_UpperCamelCase    ,		len(_UpperCamelCase       )       )
																	self.assertEqual(_UpperCamelCase    ,		all_size_a + len(_UpperCamelCase       )       )

																	UpperCAmelCase_   :				str						=						tokenizer.encode(
																	    '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'    ,		add_special_tokens=_UpperCamelCase       )

																	self.assertGreaterEqual(len(_UpperCamelCase       )    ,		6       )
																	self.assertGreater(tokens[0]    ,		tokenizer.vocab_size - 1       )
																	self.assertGreater(tokens[0]    ,		tokens[1]       )
																	self.assertGreater(tokens[-3]    ,		tokenizer.vocab_size - 1       )
																	self.assertGreater(tokens[-3]    ,		tokens[-4]       )
																	self.assertEqual(tokens[0]    ,		tokenizer.eos_token_id       )
																	self.assertEqual(tokens[-3]    ,		tokenizer.pad_token_id       )



		@unittest.skip('The tokenizer shouldn\'t be used to encode input IDs (except for labels), only to decode.'       )
		def  __UpperCAmelCase      (							self       )     ->							int:
							pass



		@unittest.skip('The tokenizer shouldn\'t be used to encode input IDs (except for labels), only to decode.'       )
		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							pass



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							# The default common tokenizer tests assumes that the output of `convert_tokens_to_string` is a string which
							# is not the case for Wav2Vec2PhonemeCTCTokenizer.
							UpperCAmelCase_   :				Tuple						=						self.get_tokenizers(fast=_UpperCamelCase    ,		do_lower_case=_UpperCamelCase       )
							for tokenizer in tokenizers:
												with self.subTest(f"{tokenizer.__class__.__name__}"       ):
																	UpperCAmelCase_   :				List[str]						=						['ð', 'ɪ', 's', 'ɪ', 'z', 'ɐ', 't', 'ɛ', 'k', 's', 't']
																	UpperCAmelCase_   :				str						=						tokenizer.convert_tokens_to_string(_UpperCamelCase       )

																	self.assertIsInstance(output['text']    ,		_UpperCamelCase       )














import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						'▁'

__UpperCAmelCase     							=						{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCAmelCase     							=						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__UpperCAmelCase     							=						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
__UpperCAmelCase     							=						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   List[Any]							=  VOCAB_FILES_NAMES
		_snake_case   :   Union[str, Any]							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   Tuple							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   Any							=  ['''input_ids''', '''attention_mask''']

		_snake_case   :   List[int]							=  []
		_snake_case   :   List[int]							=  []



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<mask>"    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase = None    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							List[str]:
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_   :				List[Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else mask_token

							UpperCAmelCase_   :				Optional[int]						=						{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_UpperCamelCase    ,		eos_token=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		sep_token=_UpperCamelCase    ,		cls_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		mask_token=_UpperCamelCase    ,		tokenizer_file=_UpperCamelCase    ,		src_lang=_UpperCamelCase    ,		tgt_lang=_UpperCamelCase    ,		additional_special_tokens=_UpperCamelCase    ,		sp_model_kwargs=self.sp_model_kwargs    ,		**_UpperCamelCase    ,		)

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.Load(str(_UpperCamelCase       )       )
							UpperCAmelCase_   :				int						=						vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCAmelCase_   :				List[str]						=						{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCAmelCase_   :				str						=						1

							UpperCAmelCase_   :				Optional[int]						=						len(self.sp_model       )
							UpperCAmelCase_   :				str						=						{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCamelCase       )
							}
							UpperCAmelCase_   :				int						=						{v: k for k, v in self.lang_code_to_id.items()}
							UpperCAmelCase_   :				List[Any]						=						len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
							UpperCAmelCase_   :				Tuple						=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							UpperCAmelCase_   :				int						=						list(self.lang_code_to_id.keys()       )

							if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												self._additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in self._additional_special_tokens]       )

							UpperCAmelCase_   :				Any						=						src_lang if src_lang is not None else 'en_XX'
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[self._src_lang]
							UpperCAmelCase_   :				Dict						=						tgt_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def __getstate__(							self       )     ->							Dict:
							UpperCAmelCase_   :				Optional[int]						=						self.__dict__.copy()
							UpperCAmelCase_   :				str						=						None
							UpperCAmelCase_   :				str						=						self.sp_model.serialized_model_proto()
							return state



		def __setstate__(							self    ,		_UpperCamelCase       )     ->							Dict:
							UpperCAmelCase_   :				Union[str, Any]						=						d

							# for backward compatibility
							if not hasattr(self    ,		'sp_model_kwargs'       ):
												UpperCAmelCase_   :				Any						=						{}

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token



		@property
		def  __UpperCAmelCase      (							self       )     ->							str:
							return self._src_lang



		@src_lang.setter
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Tuple						=						new_src_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None    ,		_UpperCamelCase = False       )     ->							List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase    ,		token_ids_a=_UpperCamelCase    ,		already_has_special_tokens=_UpperCamelCase       )

							UpperCAmelCase_   :				Tuple						=						[1] * len(self.prefix_tokens       )
							UpperCAmelCase_   :				Dict						=						[1] * len(self.suffix_tokens       )
							if token_ids_a is None:
												return prefix_ones + ([0] * len(_UpperCamelCase       )) + suffix_ones
							return prefix_ones + ([0] * len(_UpperCamelCase       )) + ([0] * len(_UpperCamelCase       )) + suffix_ones



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							UpperCAmelCase_   :				int						=						[self.sep_token_id]
							UpperCAmelCase_   :				Optional[int]						=						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							int:
							if src_lang is None or tgt_lang is None:
												raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'       )
							UpperCAmelCase_   :				Optional[int]						=						src_lang
							UpperCAmelCase_   :				Dict						=						self(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase    ,		return_tensors=_UpperCamelCase    ,		**_UpperCamelCase       )
							UpperCAmelCase_   :				int						=						self.convert_tokens_to_ids(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						tgt_lang_id
							return inputs



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				str						=						{self.convert_ids_to_tokens(_UpperCamelCase       ): i for i in range(self.vocab_size       )}
							vocab.update(self.added_tokens_encoder       )
							return vocab



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							List[str]:
							return self.sp_model.encode(_UpperCamelCase    ,		out_type=_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Tuple:
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCAmelCase_   :				Dict						=						self.sp_model.PieceToId(_UpperCamelCase       )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							UpperCAmelCase_   :				Optional[int]						=						''.join(_UpperCamelCase       ).replace(_UpperCamelCase    ,		' '       ).strip()
							return out_string



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase       ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
												return
							UpperCAmelCase_   :				List[Any]						=						os.path.join(
							    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

							if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
												copyfile(self.vocab_file    ,		_UpperCamelCase       )
							elif not os.path.isfile(self.vocab_file       ):
												with open(_UpperCamelCase    ,		'wb'       ) as fi:
																	UpperCAmelCase_   :				Optional[Any]						=						self.sp_model.serialized_model_proto()
																	fi.write(_UpperCamelCase       )

							return (out_vocab_file,)



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = "en_XX"    ,		_UpperCamelCase = None    ,		_UpperCamelCase = "ro_RO"    ,		**_UpperCamelCase    ,		)     ->							BatchEncoding:
							UpperCAmelCase_   :				Union[str, Any]						=						src_lang
							UpperCAmelCase_   :				Dict						=						tgt_lang
							return super().prepare_seqaseq_batch(_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							return self.set_src_lang_special_tokens(self.src_lang       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[src_lang]
							UpperCAmelCase_   :				Optional[Any]						=						[]
							UpperCAmelCase_   :				str						=						[self.eos_token_id, self.cur_lang_code]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[lang]
							UpperCAmelCase_   :				Dict						=						[]
							UpperCAmelCase_   :				int						=						[self.eos_token_id, self.cur_lang_code]






