






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase     							=						{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__UpperCAmelCase     							=						{
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__UpperCAmelCase     							=						{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   int							=  VOCAB_FILES_NAMES
		_snake_case   :   Dict							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   Tuple							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   str							=  PRETRAINED_INIT_CONFIGURATION
		_snake_case   :   Optional[int]							=  RoFormerTokenizer



		def __init__(							self    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=True    ,		_UpperCamelCase="[UNK]"    ,		_UpperCamelCase="[SEP]"    ,		_UpperCamelCase="[PAD]"    ,		_UpperCamelCase="[CLS]"    ,		_UpperCamelCase="[MASK]"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							Tuple:
							super().__init__(
							    _UpperCamelCase    ,		tokenizer_file=_UpperCamelCase    ,		do_lower_case=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		sep_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		cls_token=_UpperCamelCase    ,		mask_token=_UpperCamelCase    ,		tokenize_chinese_chars=_UpperCamelCase    ,		strip_accents=_UpperCamelCase    ,		**_UpperCamelCase    ,		)

							UpperCAmelCase_   :				str						=						json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
							if (
							    pre_tok_state.get('lowercase'    ,		_UpperCamelCase       ) != do_lower_case
							    or pre_tok_state.get('strip_accents'    ,		_UpperCamelCase       ) != strip_accents
							):
												UpperCAmelCase_   :				Optional[Any]						=						getattr(_UpperCamelCase    ,		pre_tok_state.pop('type'       )       )
												UpperCAmelCase_   :				int						=						do_lower_case
												UpperCAmelCase_   :				Union[str, Any]						=						strip_accents
												UpperCAmelCase_   :				Optional[int]						=						pre_tok_class(**_UpperCamelCase       )

							UpperCAmelCase_   :				Tuple						=						do_lower_case



		def __getstate__(							self       )     ->							Dict:
							UpperCAmelCase_   :				List[str]						=						self.__dict__.copy()
							UpperCAmelCase_   :				Union[str, Any]						=						BertPreTokenizer()
							return state



		def __setstate__(							self    ,		_UpperCamelCase       )     ->							Optional[Any]:
							UpperCAmelCase_   :				int						=						d
							UpperCAmelCase_   :				str						=						self.__dict__['_tokenizer'].get_vocab()
							UpperCAmelCase_   :				Union[str, Any]						=						PreTokenizer.custom(JiebaPreTokenizer(_UpperCamelCase       )       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase=None       )     ->							Tuple:
							UpperCAmelCase_   :				Tuple						=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							UpperCAmelCase_   :				str						=						[self.sep_token_id]
							UpperCAmelCase_   :				Any						=						[self.cls_token_id]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							UpperCAmelCase_   :				str						=						self._tokenizer.model.save(_UpperCamelCase    ,		name=_UpperCamelCase       )
							return tuple(_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=False    ,		**_UpperCamelCase    ,		)     ->							Dict:
							UpperCAmelCase_   :				int						=						BertPreTokenizer()
							return super().save_pretrained(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )














def 					lowercase__				(					__snake_case   :							int  ):


					'''simple docstring'''


					UpperCAmelCase_   :				Tuple						=						[1]

					UpperCAmelCase_	,  UpperCAmelCase_	,  UpperCAmelCase_   :				Tuple						=						0, 0, 0
					UpperCAmelCase_   :				Union[str, Any]						=						ugly_nums[ia] * 2
					UpperCAmelCase_   :				Tuple						=						ugly_nums[ia] * 3
					UpperCAmelCase_   :				Union[str, Any]						=						ugly_nums[ia] * 5

					for _ in range(1					,	__snake_case  ):
										UpperCAmelCase_   :				Tuple						=						min(__snake_case					,	__snake_case					,	__snake_case  )
										ugly_nums.append(__snake_case  )
										if next_num == next_a:
															ia += 1
															UpperCAmelCase_   :				Union[str, Any]						=						ugly_nums[ia] * 2
										if next_num == next_a:
															ia += 1
															UpperCAmelCase_   :				Any						=						ugly_nums[ia] * 3
										if next_num == next_a:
															ia += 1
															UpperCAmelCase_   :				List[str]						=						ugly_nums[ia] * 5
					return ugly_nums[-1]


if __name__ == "__main__":
		from doctest import testmod

		testmod(verbose=True)
		print(F'{ugly_numbers(200) = }')






