import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast

class 		UpperCamelCase      (   unittest.TestCase						):



				def  _lowercase					(self    :				str)   ->     Any:
					__snake_case      :							Optional[int]						=			mock.Mock()
					__snake_case      :							Any						=			5_00
					__snake_case      :							str						=			{}
					__snake_case      :							int						=			HTTPError
					__snake_case      :							Any						=			{}

					# Download this model to make sure it's in the cache.
					__snake_case      :							Tuple						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'     ,  return_value=UpperCamelCase__) as mock_head:
						__snake_case      :							Any						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
						# This check we did call the fake head request
						mock_head.assert_called()



				@require_tokenizers
				def  _lowercase					(self    :				Optional[int])   ->     Union[str, Any]:
					__snake_case      :							str						=			mock.Mock()
					__snake_case      :							Tuple						=			5_00
					__snake_case      :							Dict						=			{}
					__snake_case      :							List[Any]						=			HTTPError
					__snake_case      :							Optional[int]						=			{}

					# Download this model to make sure it's in the cache.
					__snake_case      :							Optional[Any]						=			GPTaTokenizerFast.from_pretrained('gpt2')

					# Under the mock environment we get a 500 error when trying to reach the tokenizer.
					with mock.patch('requests.Session.request'     ,  return_value=UpperCamelCase__) as mock_head:
						__snake_case      :							str						=			GPTaTokenizerFast.from_pretrained('gpt2')
						# This check we did call the fake head request
						mock_head.assert_called()



				def  _lowercase					(self    :				Any)   ->     Union[str, Any]:







					try:
						__snake_case      :							Any						=			tempfile.mktemp()
						with open(UpperCamelCase__     ,  'wb') as f:
							http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'     ,  UpperCamelCase__)

						__snake_case      :							List[str]						=			AlbertTokenizer.from_pretrained(UpperCamelCase__)
					finally:
						os.remove(UpperCamelCase__)

					# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
					# the current folder and have the right name.
					if os.path.isfile('tokenizer.json'):
						# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
						return
					try:
						with open('tokenizer.json'     ,  'wb') as f:
							http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'     ,  UpperCamelCase__)
						__snake_case      :							Any						=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
						# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
						self.assertEqual(tokenizer.vocab_size     ,  10_00)
						# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

					finally:
						os.remove('tokenizer.json')






				def  _lowercase					(self    :				Tuple)   ->     List[Any]:
					__snake_case      :							List[str]						=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')







@is_staging_test
class 		UpperCamelCase      (   unittest.TestCase						):
				UpperCAmelCase    :				List[str]   =		["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]



				@classmethod
				def  _lowercase					(cls    :				int)   ->     Optional[int]:
					__snake_case      :							List[Any]						=			TOKEN
					HfFolder.save_token(UpperCamelCase__)



				@classmethod
				def  _lowercase					(cls    :				str)   ->     str:







					try:
						delete_repo(token=cls._token     ,  repo_id='test-tokenizer')
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token     ,  repo_id='valid_org/test-tokenizer-org')
					except HTTPError:
						pass

					try:
						delete_repo(token=cls._token     ,  repo_id='test-dynamic-tokenizer')
					except HTTPError:
						pass



				def  _lowercase					(self    :				str)   ->     List[Any]:







					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case      :							Any						=			os.path.join(UpperCamelCase__     ,  'vocab.txt')
						with open(UpperCamelCase__     ,  'w'     ,  encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case      :							Dict						=			BertTokenizer(UpperCamelCase__)

					tokenizer.push_to_hub('test-tokenizer'     ,  use_auth_token=self._token)
					__snake_case      :							int						=			BertTokenizer.from_pretrained(f"{USER}/test-tokenizer")
					self.assertDictEqual(new_tokenizer.vocab     ,  tokenizer.vocab)

					# Reset repo
					delete_repo(token=self._token     ,  repo_id='test-tokenizer')

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(UpperCamelCase__     ,  repo_id='test-tokenizer'     ,  push_to_hub=UpperCamelCase__     ,  use_auth_token=self._token)

					__snake_case      :							Optional[Any]						=			BertTokenizer.from_pretrained(f"{USER}/test-tokenizer")
					self.assertDictEqual(new_tokenizer.vocab     ,  tokenizer.vocab)



				def  _lowercase					(self    :				Optional[Any])   ->     Union[str, Any]:







					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case      :							Optional[int]						=			os.path.join(UpperCamelCase__     ,  'vocab.txt')
						with open(UpperCamelCase__     ,  'w'     ,  encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case      :							Tuple						=			BertTokenizer(UpperCamelCase__)

					tokenizer.push_to_hub('valid_org/test-tokenizer-org'     ,  use_auth_token=self._token)
					__snake_case      :							str						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
					self.assertDictEqual(new_tokenizer.vocab     ,  tokenizer.vocab)

					# Reset repo
					delete_repo(token=self._token     ,  repo_id='valid_org/test-tokenizer-org')

					# Push to hub via save_pretrained
					with tempfile.TemporaryDirectory() as tmp_dir:
						tokenizer.save_pretrained(
						    UpperCamelCase__     ,  repo_id='valid_org/test-tokenizer-org'     ,  push_to_hub=UpperCamelCase__     ,  use_auth_token=self._token)

					__snake_case      :							Optional[int]						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
					self.assertDictEqual(new_tokenizer.vocab     ,  tokenizer.vocab)






				@require_tokenizers
				def  _lowercase					(self    :				str)   ->     Optional[int]:
					CustomTokenizer.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case      :							Dict						=			os.path.join(UpperCamelCase__     ,  'vocab.txt')
						with open(UpperCamelCase__     ,  'w'     ,  encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
						__snake_case      :							int						=			CustomTokenizer(UpperCamelCase__)

					# No fast custom tokenizer
					tokenizer.push_to_hub('test-dynamic-tokenizer'     ,  use_auth_token=self._token)

					__snake_case      :							Any						=			AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,  trust_remote_code=UpperCamelCase__)
					# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__     ,  'CustomTokenizer')

					# Fast and slow custom tokenizer
					CustomTokenizerFast.register_for_auto_class()
					with tempfile.TemporaryDirectory() as tmp_dir:
						__snake_case      :							int						=			os.path.join(UpperCamelCase__     ,  'vocab.txt')
						with open(UpperCamelCase__     ,  'w'     ,  encoding='utf-8') as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

						__snake_case      :							Optional[Any]						=			BertTokenizerFast.from_pretrained(UpperCamelCase__)
						bert_tokenizer.save_pretrained(UpperCamelCase__)
						__snake_case      :							List[Any]						=			CustomTokenizerFast.from_pretrained(UpperCamelCase__)

					tokenizer.push_to_hub('test-dynamic-tokenizer'     ,  use_auth_token=self._token)

					__snake_case      :							str						=			AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"     ,  trust_remote_code=UpperCamelCase__)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__     ,  'CustomTokenizerFast')
					__snake_case      :							Any						=			AutoTokenizer.from_pretrained(
					    f"{USER}/test-dynamic-tokenizer"     ,  use_fast=UpperCamelCase__     ,  trust_remote_code=UpperCamelCase__)
					# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
					self.assertEqual(tokenizer.__class__.__name__     ,  'CustomTokenizer')







class 		UpperCamelCase      (   unittest.TestCase						):



				def  _lowercase					(self    :				Dict)   ->     Union[str, Any]:
					__snake_case      :							List[Any]						=			Trie()
					trie.add('Hello 友達')
					self.assertEqual(trie.data     ,  {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
					trie.add('Hello')
					trie.data
					self.assertEqual(trie.data     ,  {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})



				def  _lowercase					(self    :				List[str])   ->     int:
					__snake_case      :							Tuple						=			Trie()
					self.assertEqual(trie.split('[CLS] This is a extra_id_100')     ,  ['[CLS] This is a extra_id_100'])
					trie.add('[CLS]')
					trie.add('extra_id_1')
					trie.add('extra_id_100')
					self.assertEqual(trie.split('[CLS] This is a extra_id_100')     ,  ['[CLS]', ' This is a ', 'extra_id_100'])



				def  _lowercase					(self    :				str)   ->     int:
					__snake_case      :							str						=			Trie()
					trie.add('A')
					self.assertEqual(trie.split('ABC')     ,  ['A', 'BC'])
					self.assertEqual(trie.split('BCA')     ,  ['BC', 'A'])



				def  _lowercase					(self    :				Optional[Any])   ->     Dict:
					__snake_case      :							Any						=			Trie()
					trie.add('TOKEN]')
					trie.add('[SPECIAL_TOKEN]')
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')     ,  ['This is something ', '[SPECIAL_TOKEN]'])



				def  _lowercase					(self    :				Union[str, Any])   ->     Optional[int]:
					__snake_case      :							Tuple						=			Trie()
					trie.add('A')
					trie.add('P')
					trie.add('[SPECIAL_TOKEN]')
					self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')     ,  ['This is something ', '[SPECIAL_TOKEN]'])



				def  _lowercase					(self    :				Optional[int])   ->     Tuple:
					__snake_case      :							Any						=			Trie()
					trie.add('AB')
					trie.add('B')
					trie.add('C')
					self.assertEqual(trie.split('ABC')     ,  ['AB', 'C'])



				def  _lowercase					(self    :				List[str])   ->     Dict:
					__snake_case      :							str						=			Trie()
					trie.add('ABC')
					trie.add('B')
					trie.add('CD')
					self.assertEqual(trie.split('ABCD')     ,  ['ABC', 'D'])






				def  _lowercase					(self    :				str)   ->     List[str]:
					__snake_case      :							str						=			Trie()
					__snake_case      :							List[Any]						=			trie.cut_text('ABC'     ,  [0, 0, 2, 1, 2, 3])
					self.assertEqual(UpperCamelCase__     ,  ['AB', 'C'])






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_a					:					List[Any]=			logging.get_logger(__name__)

_a					:					Any=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					int=			{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

_a					:					Optional[Any]=			{
    "junnyu/roformer_chinese_small": 1_536,
    "junnyu/roformer_chinese_base": 1_536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


_a					:					str=			{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Union[str, Any]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Optional[int]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[str]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Dict   =		RoFormerTokenizer



				def __init__(self    :				List[Any]     ,  _A    :				Any=None     ,  _A    :				int=None     ,  _A    :				Dict=True     ,  _A    :				List[Any]="[UNK]"     ,  _A    :				Tuple="[SEP]"     ,  _A    :				List[Any]="[PAD]"     ,  _A    :				str="[CLS]"     ,  _A    :				int="[MASK]"     ,  _A    :				Optional[int]=True     ,  _A    :				List[str]=None     ,  **_A    :				int     ,  )   ->     Dict:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  **_A     ,  )

					__snake_case      :							List[Any]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    pre_tok_state.get('lowercase'     ,  _A) != do_lower_case
					    or pre_tok_state.get('strip_accents'     ,  _A) != strip_accents
					):
						__snake_case      :							Union[str, Any]						=			getattr(_A     ,  pre_tok_state.pop('type'))
						__snake_case      :							Union[str, Any]						=			do_lower_case
						__snake_case      :							str						=			strip_accents
						__snake_case      :							Optional[int]						=			pre_tok_class(**_A)

					__snake_case      :							int						=			do_lower_case



				def __getstate__(self    :				Optional[Any])   ->     Dict:
					__snake_case      :							Optional[int]						=			self.__dict__.copy()
					__snake_case      :							int						=			BertPreTokenizer()
					return state



				def __setstate__(self    :				Optional[Any]     ,  _A    :				Optional[Any])   ->     Dict:
					__snake_case      :							List[str]						=			d
					__snake_case      :							str						=			self.__dict__['_tokenizer'].get_vocab()
					__snake_case      :							int						=			PreTokenizer.custom(JiebaPreTokenizer(_A))



				def  _lowercase					(self    :				int     ,  _A    :				Tuple     ,  _A    :				Any=None)   ->     str:
					__snake_case      :							Dict						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				List[str]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Tuple						=			[self.sep_token_id]
					__snake_case      :							Optional[Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



				def  _lowercase					(self    :				List[Any]     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							List[Any]						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)






				def  _lowercase					(self    :				int     ,  _A    :				Optional[int]     ,  _A    :				Tuple=None     ,  _A    :				Tuple=None     ,  _A    :				Dict=False     ,  **_A    :				Optional[int]     ,  )   ->     Optional[Any]:
					__snake_case      :							Optional[Any]						=			BertPreTokenizer()
					return super().save_pretrained(_A     ,  _A     ,  _A     ,  _A     ,  **_A)





