

import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     {
    '''vocab_file''': '''vocab.txt''',
    '''merges_file''': '''bpe.codes''',
}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt''',
    },
    '''merges_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes''',
    },
}

UpperCamelCase			    =     {
    '''vinai/phobert-base''': 256,
    '''vinai/phobert-large''': 256,
}

def 	__lowerCamelCase      (		snake_case__		)							->  Any:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  set()
						_SCREAMING_SNAKE_CASE     				=  word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)		)
												_SCREAMING_SNAKE_CASE     				=  char

						_SCREAMING_SNAKE_CASE     				=  set(snake_case__		)
						return pairs




class 				__UpperCAmelCase		(snake_case_   ):
			__snake_case    :				Optional[Any]									=							VOCAB_FILES_NAMES
			__snake_case    :				List[Any]									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				str									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







			def __init__(     self:  List[Any]				,     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Dict="<s>"				,     UpperCAmelCase_:  Optional[Any]="</s>"				,     UpperCAmelCase_:  Dict="</s>"				,     UpperCAmelCase_:  Dict="<s>"				,     UpperCAmelCase_:  Dict="<unk>"				,     UpperCAmelCase_:  List[Any]="<pad>"				,     UpperCAmelCase_:  Optional[Any]="<mask>"				,     **UpperCAmelCase_:  str				,     ):







									'''simple docstring'''




									super().__init__(
									    bos_token=UpperCAmelCase_				,     eos_token=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     mask_token=UpperCAmelCase_				,     **UpperCAmelCase_				,     )

									_SCREAMING_SNAKE_CASE     				=  vocab_file
									_SCREAMING_SNAKE_CASE     				=  merges_file

									_SCREAMING_SNAKE_CASE     				=  {}
									_SCREAMING_SNAKE_CASE     				=  0
									_SCREAMING_SNAKE_CASE     				=  1
									_SCREAMING_SNAKE_CASE     				=  2
									_SCREAMING_SNAKE_CASE     				=  3

									self.add_from_file(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  {v: k for k, v in self.encoder.items()}

									with open(UpperCAmelCase_				,     encoding="""utf-8"""					) as merges_handle:
															_SCREAMING_SNAKE_CASE     				=  merges_handle.read().split("""\n"""					)[:-1]
									_SCREAMING_SNAKE_CASE     				=  [tuple(merge.split()[:-1]					) for merge in merges]
									_SCREAMING_SNAKE_CASE     				=  dict(zip(UpperCAmelCase_				,     range(len(UpperCAmelCase_					)					)					)					)
									_SCREAMING_SNAKE_CASE     				=  {}







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None				,     UpperCAmelCase_:  bool = False					):







									'''simple docstring'''




									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCAmelCase_				,     token_ids_a=UpperCAmelCase_				,     already_has_special_tokens=UpperCAmelCase_					)

									if token_ids_a is None:
															return [1] + ([0] * len(UpperCAmelCase_					)) + [1]
									return [1] + ([0] * len(UpperCAmelCase_					)) + [1, 1] + ([0] * len(UpperCAmelCase_					)) + [1]







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







			@property
			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									return len(self.encoder					)







			def 							UpperCamelCase				(     self:  int					):







									'''simple docstring'''




									return dict(self.encoder				,     **self.added_tokens_encoder					)







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Tuple					):







									'''simple docstring'''




									if token in self.cache:
															return self.cache[token]
									_SCREAMING_SNAKE_CASE     				=  tuple(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  tuple(list(word[:-1]					) + [word[-1] + """</w>"""]					)
									_SCREAMING_SNAKE_CASE     				=  get_pairs(UpperCAmelCase_					)

									if not pairs:
															return token

									while True:
															_SCREAMING_SNAKE_CASE     				=  min(UpperCAmelCase_				,     key=lambda UpperCAmelCase_					: self.bpe_ranks.get(UpperCAmelCase_				,     float("""inf"""					)					)					)
															if bigram not in self.bpe_ranks:
																					break
															_SCREAMING_SNAKE_CASE     				=  bigram
															_SCREAMING_SNAKE_CASE     				=  []
															_SCREAMING_SNAKE_CASE     				=  0
															while i < len(UpperCAmelCase_					):
																					try:
																											_SCREAMING_SNAKE_CASE     				=  word.index(UpperCAmelCase_				,     UpperCAmelCase_					)
																					except ValueError:
																											new_word.extend(word[i:]					)
																											break
																					else:
																											new_word.extend(word[i:j]					)
																											_SCREAMING_SNAKE_CASE     				=  j

																					if word[i] == first and i < len(UpperCAmelCase_					) - 1 and word[i + 1] == second:
																											new_word.append(first + second					)
																											i += 2
																					else:
																											new_word.append(word[i]					)
																											i += 1
															_SCREAMING_SNAKE_CASE     				=  tuple(UpperCAmelCase_					)
															_SCREAMING_SNAKE_CASE     				=  new_word
															if len(UpperCAmelCase_					) == 1:
																					break
															else:
																					_SCREAMING_SNAKE_CASE     				=  get_pairs(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  '@@ '.join(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  word[:-4]
									_SCREAMING_SNAKE_CASE     				=  word
									return word







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  []

									_SCREAMING_SNAKE_CASE     				=  re.findall(R"""\S+\n?"""				,     UpperCAmelCase_					)

									for token in words:
															split_tokens.extend(list(self.bpe(UpperCAmelCase_					).split(""" """					)					)					)
									return split_tokens







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  Tuple					):







									'''simple docstring'''




									return self.encoder.get(UpperCAmelCase_				,     self.encoder.get(self.unk_token					)					)







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									return self.decoder.get(UpperCAmelCase_				,     self.unk_token					)







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ' '.join(UpperCAmelCase_					).replace("""@@ """				,     """"""					).strip()
									return out_string







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									if not os.path.isdir(UpperCAmelCase_					):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
															return
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCAmelCase_					):
															copyfile(self.vocab_file				,     UpperCAmelCase_					)

									if os.path.abspath(self.merges_file					) != os.path.abspath(UpperCAmelCase_					):
															copyfile(self.merges_file				,     UpperCAmelCase_					)

									return out_vocab_file, out_merge_file







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
															try:
																					with open(UpperCAmelCase_				,     """r"""				,     encoding="""utf-8"""					) as fd:
																											self.add_from_file(UpperCAmelCase_					)
															except FileNotFoundError as fnfe:
																					raise fnfe
															except UnicodeError:
																					raise Exception(F'Incorrect encoding detected in {f}, please rebuild the dataset'					)
															return

									_SCREAMING_SNAKE_CASE     				=  f.readlines()
									for lineTmp in lines:
															_SCREAMING_SNAKE_CASE     				=  lineTmp.strip()
															_SCREAMING_SNAKE_CASE     				=  line.rfind(""" """					)
															if idx == -1:
																					raise ValueError("""Incorrect dictionary format, expected \'<token> <cnt>\'"""					)
															_SCREAMING_SNAKE_CASE     				=  line[:idx]
															_SCREAMING_SNAKE_CASE     				=  len(self.encoder					)









import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 				__UpperCAmelCase		(_UpperCAmelCase	,unittest.TestCase   ):
			__snake_case    :				Optional[Any]									=							OpenAIGPTTokenizer
			__snake_case    :				Dict									=							OpenAIGPTTokenizerFast
			__snake_case    :				Optional[Any]									=							True
			__snake_case    :				Dict									=							False







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									_SCREAMING_SNAKE_CASE     				=  [
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """w</w>""",
									    """r</w>""",
									    """t</w>""",
									    """lo""",
									    """low""",
									    """er</w>""",
									    """low</w>""",
									    """lowest</w>""",
									    """newer</w>""",
									    """wider</w>""",
									    """<unk>""",
									]
									_SCREAMING_SNAKE_CASE     				=  dict(zip(UpperCAmelCase_				,     range(len(UpperCAmelCase_					)					)					)					)
									_SCREAMING_SNAKE_CASE     				=  ["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""vocab_file"""]					)
									_SCREAMING_SNAKE_CASE     				=  os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES["""merges_file"""]					)
									with open(self.vocab_file				,     """w"""					) as fp:
															fp.write(json.dumps(UpperCAmelCase_					)					)
									with open(self.merges_file				,     """w"""					) as fp:
															fp.write("""\n""".join(UpperCAmelCase_					)					)







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Dict					):







									'''simple docstring'''




									return "lower newer", "lower newer"







			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  OpenAIGPTTokenizer(self.vocab_file				,     self.merges_file					)

									_SCREAMING_SNAKE_CASE     				=  """lower"""
									_SCREAMING_SNAKE_CASE     				=  ["""low""", """er</w>"""]
									_SCREAMING_SNAKE_CASE     				=  tokenizer.tokenize(UpperCAmelCase_					)
									self.assertListEqual(UpperCAmelCase_				,     UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  tokens + ["""<unk>"""]
									_SCREAMING_SNAKE_CASE     				=  [14, 15, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_					)				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Union[str, Any]=15					):







									'''simple docstring'''




									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																					_SCREAMING_SNAKE_CASE     				=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_				,     **UpperCAmelCase_					)

																					# Simple input
																					_SCREAMING_SNAKE_CASE     				=  """This is a simple input"""
																					_SCREAMING_SNAKE_CASE     				=  ["""This is a simple input 1""", """This is a simple input 2"""]
																					_SCREAMING_SNAKE_CASE     				=  ("""This is a simple input""", """This is a pair""")
																					_SCREAMING_SNAKE_CASE     				=  [
																					    ("""This is a simple input 1""", """This is a simple input 2"""),
																					    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																					]

																					# Simple input tests
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Simple input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(UpperCAmelCase_				,     tokenizer_r.encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""					)

																					# Pair input
																					self.assertRaises(
																					    UpperCAmelCase_				,     tokenizer_r.batch_encode_plus				,     UpperCAmelCase_				,     max_length=UpperCAmelCase_				,     padding="""max_length"""				,     )







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									pass



@require_ftfy
@require_spacy
@require_tokenizers
class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			pass

