


import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase	=							logging.get_logger(__name__)

lowerCAmelCase	=							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase	=							{
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

lowerCAmelCase	=							{
    'ctrl': 256,
}

lowerCAmelCase	=							{
    'Pregnancy': 16_8629,
    'Christianity': 7675,
    'Explain': 10_6423,
    'Fitness': 6_3440,
    'Saving': 6_3163,
    'Ask': 2_7171,
    'Ass': 9_5985,
    'Joke': 16_3509,
    'Questions': 4_5622,
    'Thoughts': 4_9605,
    'Retail': 5_2342,
    'Feminism': 16_4338,
    'Writing': 1_1992,
    'Atheism': 19_2263,
    'Netflix': 4_8616,
    'Computing': 3_9639,
    'Opinion': 4_3213,
    'Alone': 4_4967,
    'Funny': 5_8917,
    'Gaming': 4_0358,
    'Human': 4088,
    'India': 1331,
    'Joker': 7_7138,
    'Diet': 3_6206,
    'Legal': 1_1859,
    'Norman': 4939,
    'Tip': 7_2689,
    'Weight': 5_2343,
    'Movies': 4_6273,
    'Running': 2_3425,
    'Science': 2090,
    'Horror': 3_7793,
    'Confession': 6_0572,
    'Finance': 1_2250,
    'Politics': 1_6360,
    'Scary': 19_1985,
    'Support': 1_2654,
    'Technologies': 3_2516,
    'Teenage': 6_6160,
    'Event': 3_2769,
    'Learned': 6_7460,
    'Notion': 18_2770,
    'Wikipedia': 3_7583,
    'Books': 6665,
    'Extract': 7_6050,
    'Confessions': 10_2701,
    'Conspiracy': 7_5932,
    'Links': 6_3674,
    'Narcissus': 15_0425,
    'Relationship': 5_4766,
    'Relationships': 13_4796,
    'Reviews': 4_1671,
    'News': 4256,
    'Translation': 2_6820,
    'multilingual': 12_8406,
}
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		lowercase__						=				set()
		lowercase__						=				word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)					)
				lowercase__						=				char

		lowercase__						=				set(SCREAMING_SNAKE_CASE					)
		return pairs


class 	_a							(      UpperCamelCase__      ):
							_lowercase     :      Any  				=			VOCAB_FILES_NAMES
							_lowercase     :      List[str]  				=			PRETRAINED_VOCAB_FILES_MAP
							_lowercase     :      Dict  				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase     :      Any  				=			CONTROL_CODES

							def __init__(							self:     Optional[int]   ,						UpperCamelCase_:     Optional[int]   ,						UpperCamelCase_:     List[Any]   ,						UpperCamelCase_:     int="<unk>"   ,						**UpperCamelCase_:     Union[str, Any]							)      ->       List[str]:





									"""simple docstring"""







									super().__init__(unk_token=UpperCamelCase_   ,						**UpperCamelCase_							)

									with open(UpperCamelCase_   ,						encoding='''utf-8'''							) as vocab_handle:
											lowercase__						=				json.load(UpperCamelCase_							)
									lowercase__						=				{v: k for k, v in self.encoder.items()}
									with open(UpperCamelCase_   ,						encoding='''utf-8'''							) as merges_handle:
											lowercase__						=				merges_handle.read().split('''\n'''							)[1:-1]
									lowercase__						=				[tuple(merge.split()							) for merge in merges]
									lowercase__						=				dict(zip(UpperCamelCase_   ,						range(len(UpperCamelCase_							)							)							)							)
									lowercase__						=				{}

							@property
							def 				lowerCamelCase_			(							self:     str							)      ->       Union[str, Any]:





									"""simple docstring"""







									return len(self.encoder							)

							def 				lowerCamelCase_			(							self:     int							)      ->       Tuple:





									"""simple docstring"""







									return dict(self.encoder   ,						**self.added_tokens_encoder							)

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Any							)      ->       List[str]:





									"""simple docstring"""







									if token in self.cache:
											return self.cache[token]
									lowercase__						=				tuple(UpperCamelCase_							)
									lowercase__						=				tuple(list(word[:-1]							) + [word[-1] + '''</w>''']							)
									lowercase__						=				get_pairs(UpperCamelCase_							)

									if not pairs:
											return token

									while True:
											lowercase__						=				min(UpperCamelCase_   ,						key=lambda UpperCamelCase_							: self.bpe_ranks.get(UpperCamelCase_   ,						float('''inf'''							)							)							)
											if bigram not in self.bpe_ranks:
													break
											lowercase__					,		lowercase__						=				bigram
											lowercase__						=				[]
											lowercase__						=				0
											while i < len(UpperCamelCase_							):
													try:
															lowercase__						=				word.index(UpperCamelCase_   ,						UpperCamelCase_							)
													except ValueError:
															new_word.extend(word[i:]							)
															break
													else:
															new_word.extend(word[i:j]							)
															lowercase__						=				j

													if word[i] == first and i < len(UpperCamelCase_							) - 1 and word[i + 1] == second:
															new_word.append(first + second							)
															i += 2
													else:
															new_word.append(word[i]							)
															i += 1
											lowercase__						=				tuple(UpperCamelCase_							)
											lowercase__						=				new_word
											if len(UpperCamelCase_							) == 1:
													break
											else:
													lowercase__						=				get_pairs(UpperCamelCase_							)
									lowercase__						=				'''@@ '''.join(UpperCamelCase_							)
									lowercase__						=				word[:-4]
									lowercase__						=				word
									return word

							def 				lowerCamelCase_			(							self:     str   ,						UpperCamelCase_:     Union[str, Any]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				[]

									lowercase__						=				re.findall(r'''\S+\n?'''   ,						UpperCamelCase_							)

									for token in words:
											split_tokens.extend(list(self.bpe(UpperCamelCase_							).split(''' '''							)							)							)
									return split_tokens

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Optional[Any]							)      ->       List[Any]:





									"""simple docstring"""







									return self.encoder.get(UpperCamelCase_   ,						self.encoder.get(self.unk_token							)							)

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     int							)      ->       Tuple:





									"""simple docstring"""







									return self.decoder.get(UpperCamelCase_   ,						self.unk_token							)

							def 				lowerCamelCase_			(							self:     List[Any]   ,						UpperCamelCase_:     Union[str, Any]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				''' '''.join(UpperCamelCase_							).replace('''@@ '''   ,						''''''							).strip()
									return out_string


							def 				lowerCamelCase_			(							self:     List[Any]   ,						UpperCamelCase_:     str   ,						UpperCamelCase_:     Optional[str] = None							)      ->       Tuple[str]:





									"""simple docstring"""







									if not os.path.isdir(UpperCamelCase_							):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'							)
											return
									lowercase__						=				os.path.join(
									    UpperCamelCase_   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
									lowercase__						=				os.path.join(
									    UpperCamelCase_   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']							)

									with open(UpperCamelCase_   ,						'''w'''   ,						encoding='''utf-8'''							) as f:
											f.write(json.dumps(self.encoder   ,						indent=2   ,						sort_keys=UpperCamelCase_   ,						ensure_ascii=UpperCamelCase_							) + '''\n'''							)

									lowercase__						=				0
									with open(UpperCamelCase_   ,						'''w'''   ,						encoding='''utf-8'''							) as writer:
											writer.write('''#version: 0.2\n'''							)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,						key=lambda UpperCamelCase_							: kv[1]							):
													if index != token_index:
															logger.warning(
															    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
															    ''' Please check that the tokenizer is not corrupted!'''							)
															lowercase__						=				token_index
													writer.write(''' '''.join(UpperCamelCase_							) + '''\n'''							)
													index += 1

									return vocab_file, merge_file

							# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
							#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
							#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
							#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
							#     return ''.join(tokens_generated_so_far)






def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		return base * power(SCREAMING_SNAKE_CASE						,    (exponent - 1)					) if exponent else 1


if __name__ == "__main__":
			print('Raise base to the power of exponent using recursion...')
			lowerCAmelCase	=							int(input('Enter the base: ').strip())
			lowerCAmelCase	=							int(input('Enter the exponent: ').strip())
			lowerCAmelCase	=							power(base, abs(exponent))
			if exponent < 0:  # power() does not properly deal w/ negative exponents
						lowerCAmelCase	=							1 / result
			print(f"""{base} to the power of {exponent} is {result}""")


