from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
	from transformers import (
	    TFAutoModel,
	    TFAutoModelForCausalLM,
	    TFAutoModelForMaskedLM,
	    TFAutoModelForPreTraining,
	    TFAutoModelForQuestionAnswering,
	    TFAutoModelForSeqaSeqLM,
	    TFAutoModelForSequenceClassification,
	    TFAutoModelForTableQuestionAnswering,
	    TFAutoModelForTokenClassification,
	    TFAutoModelWithLMHead,
	    TFBertForMaskedLM,
	    TFBertForPreTraining,
	    TFBertForQuestionAnswering,
	    TFBertForSequenceClassification,
	    TFBertModel,
	    TFFunnelBaseModel,
	    TFFunnelModel,
	    TFGPTaLMHeadModel,
	    TFRobertaForMaskedLM,
	    TFTaForConditionalGeneration,
	    TFTapasForQuestionAnswering,
	)
	from transformers.models.auto.modeling_tf_auto import (
	    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
	    TF_MODEL_FOR_MASKED_LM_MAPPING,
	    TF_MODEL_FOR_PRETRAINING_MAPPING,
	    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
	    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
	    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
	    TF_MODEL_MAPPING,
	)
	from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Optional[int]   =		"""new-model"""


if is_tf_available():

	class 		UpperCamelCase      (   lowercase						):
					UpperCAmelCase    :				List[str]   =		NewModelConfig

@require_tf
class 		UpperCamelCase      (   unittest.TestCase						):



				@slow
				def  _lowercase					(self    :				List[str])   ->     Dict:
					__snake_case      :							Any						=			'bert-base-cased'
					__snake_case      :							Optional[Any]						=			AutoConfig.from_pretrained(_A)
					self.assertIsNotNone(_A)
					self.assertIsInstance(_A     ,  _A)

					__snake_case      :							Union[str, Any]						=			TFAutoModel.from_pretrained(_A)
					self.assertIsNotNone(_A)
					self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				List[Any])   ->     str:
					__snake_case      :							Optional[int]						=			'bert-base-cased'
					__snake_case      :							List[Any]						=			AutoConfig.from_pretrained(_A)
					self.assertIsNotNone(_A)
					self.assertIsInstance(_A     ,  _A)

					__snake_case      :							Dict						=			TFAutoModelForPreTraining.from_pretrained(_A)
					self.assertIsNotNone(_A)
					self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				Any)   ->     List[str]:
					for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__snake_case      :							str						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							List[str]						=			TFAutoModelForCausalLM.from_pretrained(_A)
						__snake_case      :							List[str]						=			TFAutoModelForCausalLM.from_pretrained(_A     ,  output_loading_info=_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				Tuple)   ->     Dict:
					for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__snake_case      :							Union[str, Any]						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							Union[str, Any]						=			TFAutoModelWithLMHead.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				Union[str, Any])   ->     Optional[int]:
					for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__snake_case      :							Union[str, Any]						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							Optional[Any]						=			TFAutoModelForMaskedLM.from_pretrained(_A)
						__snake_case      :							Optional[int]						=			TFAutoModelForMaskedLM.from_pretrained(_A     ,  output_loading_info=_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				str)   ->     Union[str, Any]:
					for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						__snake_case      :							Dict						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							str						=			TFAutoModelForSeqaSeqLM.from_pretrained(_A)
						__snake_case      :							Union[str, Any]						=			TFAutoModelForSeqaSeqLM.from_pretrained(_A     ,  output_loading_info=_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				str)   ->     str:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
						__snake_case      :							Tuple						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							Tuple						=			TFAutoModelForSequenceClassification.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				def  _lowercase					(self    :				Optional[Any])   ->     Optional[int]:
					# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					for model_name in ["bert-base-uncased"]:
						__snake_case      :							List[str]						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							Any						=			TFAutoModelForQuestionAnswering.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				@slow
				@require_tensorflow_probability
				def  _lowercase					(self    :				List[Any])   ->     List[str]:
					for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
						__snake_case      :							Optional[Any]						=			AutoConfig.from_pretrained(_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)

						__snake_case      :							int						=			TFAutoModelForTableQuestionAnswering.from_pretrained(_A)
						__snake_case      :							Dict						=			TFAutoModelForTableQuestionAnswering.from_pretrained(
						    _A     ,  output_loading_info=_A)
						self.assertIsNotNone(_A)
						self.assertIsInstance(_A     ,  _A)



				def  _lowercase					(self    :				Optional[Any])   ->     Optional[Any]:
					__snake_case      :							Optional[int]						=			TFAutoModelWithLMHead.from_pretrained(_A)
					self.assertIsInstance(_A     ,  _A)
					self.assertEqual(model.num_parameters()     ,  1_44_10)
					self.assertEqual(model.num_parameters(only_trainable=_A)     ,  1_44_10)



				def  _lowercase					(self    :				Any)   ->     List[str]:
					__snake_case      :							List[Any]						=			TFAutoModelWithLMHead.from_pretrained(_A)
					self.assertIsInstance(_A     ,  _A)
					self.assertEqual(model.num_parameters()     ,  1_44_10)
					self.assertEqual(model.num_parameters(only_trainable=_A)     ,  1_44_10)



				def  _lowercase					(self    :				Optional[Any])   ->     str:
					# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
					__snake_case      :							Optional[Any]						=			TFAutoModel.from_pretrained('sgugger/funnel-random-tiny')
					self.assertIsInstance(_A     ,  _A)

					__snake_case      :							int						=			copy.deepcopy(model.config)
					__snake_case      :							int						=			['FunnelBaseModel']
					__snake_case      :							int						=			TFAutoModel.from_config(_A)
					self.assertIsInstance(_A     ,  _A)

					with tempfile.TemporaryDirectory() as tmp_dir:
						model.save_pretrained(_A)
						__snake_case      :							List[Any]						=			TFAutoModel.from_pretrained(_A)
						self.assertIsInstance(_A     ,  _A)



				def  _lowercase					(self    :				List[Any])   ->     int:
					try:
						AutoConfig.register('new-model'     ,  _A)

						__snake_case      :							int						=			[
						    TFAutoModel,
						    TFAutoModelForCausalLM,
						    TFAutoModelForMaskedLM,
						    TFAutoModelForPreTraining,
						    TFAutoModelForQuestionAnswering,
						    TFAutoModelForSequenceClassification,
						    TFAutoModelForTokenClassification,
						]

						for auto_class in auto_classes:
							with self.subTest(auto_class.__name__):
								# Wrong config class will raise an error
								with self.assertRaises(_A):
									auto_class.register(_A     ,  _A)
								auto_class.register(_A     ,  _A)
								# Trying to register something existing in the Transformers library will raise an error
								with self.assertRaises(_A):
									auto_class.register(_A     ,  _A)

								# Now that the config is registered, it can be used as any other config with the auto-API
								__snake_case      :							Union[str, Any]						=			BertModelTester(self).get_config()
								__snake_case      :							Optional[int]						=			NewModelConfig(**tiny_config.to_dict())
								__snake_case      :							List[str]						=			auto_class.from_config(_A)
								self.assertIsInstance(_A     ,  _A)

								with tempfile.TemporaryDirectory() as tmp_dir:
									model.save_pretrained(_A)
									__snake_case      :							Tuple						=			auto_class.from_pretrained(_A)
									self.assertIsInstance(_A     ,  _A)

					finally:
						if "new-model" in CONFIG_MAPPING._extra_content:
							del CONFIG_MAPPING._extra_content["new-model"]
						for mapping in (
						    TF_MODEL_MAPPING,
						    TF_MODEL_FOR_PRETRAINING_MAPPING,
						    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
						    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
						    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
						    TF_MODEL_FOR_MASKED_LM_MAPPING,
						):
							if NewModelConfig in mapping._extra_content:
								del mapping._extra_content[NewModelConfig]



				def  _lowercase					(self    :				Optional[int])   ->     Union[str, Any]:
					with self.assertRaisesRegex(
					    _A     ,  'bert-base is not a local folder and is not a valid model identifier'):
						__snake_case      :							Any						=			TFAutoModel.from_pretrained('bert-base')



				def  _lowercase					(self    :				str)   ->     str:
					with self.assertRaisesRegex(
					    _A     ,  r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'):
						__snake_case      :							Optional[Any]						=			TFAutoModel.from_pretrained(_A     ,  revision='aaaaaa')



				def  _lowercase					(self    :				int)   ->     Any:
					with self.assertRaisesRegex(
					    _A     ,  'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'     ,  ):
						__snake_case      :							List[str]						=			TFAutoModel.from_pretrained('hf-internal-testing/config-no-model')



				def  _lowercase					(self    :				Optional[Any])   ->     Any:
					with self.assertRaisesRegex(_A     ,  'Use `from_pt=True` to load this model'):
						__snake_case      :							List[str]						=			TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only')






				def  _lowercase					(self    :				str)   ->     Any:
					# Make sure we have cached the model.
					__snake_case      :							str						=			TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert')
					with RequestCounter() as counter:
						__snake_case      :							List[str]						=			TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert')
						self.assertEqual(counter.get_request_count     ,  0)
						self.assertEqual(counter.head_request_count     ,  1)
						self.assertEqual(counter.other_request_count     ,  0)

					# With a sharded checkpoint
					__snake_case      :							Optional[int]						=			TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded')
					with RequestCounter() as counter:
						__snake_case      :							Any						=			TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded')
						self.assertEqual(counter.get_request_count     ,  0)
						self.assertEqual(counter.head_request_count     ,  1)
						self.assertEqual(counter.other_request_count     ,  0)






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_a					:					List[Any]=			logging.get_logger(__name__)

_a					:					Any=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					int=			{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

_a					:					Optional[Any]=			{
    "junnyu/roformer_chinese_small": 1_536,
    "junnyu/roformer_chinese_base": 1_536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


_a					:					str=			{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				Union[str, Any]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Optional[int]   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[str]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Dict   =		RoFormerTokenizer



				def __init__(self    :				List[Any]     ,  _A    :				Any=None     ,  _A    :				int=None     ,  _A    :				Dict=True     ,  _A    :				List[Any]="[UNK]"     ,  _A    :				Tuple="[SEP]"     ,  _A    :				List[Any]="[PAD]"     ,  _A    :				str="[CLS]"     ,  _A    :				int="[MASK]"     ,  _A    :				Optional[int]=True     ,  _A    :				List[str]=None     ,  **_A    :				int     ,  )   ->     Dict:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  **_A     ,  )

					__snake_case      :							List[Any]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    pre_tok_state.get('lowercase'     ,  _A) != do_lower_case
					    or pre_tok_state.get('strip_accents'     ,  _A) != strip_accents
					):
						__snake_case      :							Union[str, Any]						=			getattr(_A     ,  pre_tok_state.pop('type'))
						__snake_case      :							Union[str, Any]						=			do_lower_case
						__snake_case      :							str						=			strip_accents
						__snake_case      :							Optional[int]						=			pre_tok_class(**_A)

					__snake_case      :							int						=			do_lower_case



				def __getstate__(self    :				Optional[Any])   ->     Dict:
					__snake_case      :							Optional[int]						=			self.__dict__.copy()
					__snake_case      :							int						=			BertPreTokenizer()
					return state



				def __setstate__(self    :				Optional[Any]     ,  _A    :				Optional[Any])   ->     Dict:
					__snake_case      :							List[str]						=			d
					__snake_case      :							str						=			self.__dict__['_tokenizer'].get_vocab()
					__snake_case      :							int						=			PreTokenizer.custom(JiebaPreTokenizer(_A))



				def  _lowercase					(self    :				int     ,  _A    :				Tuple     ,  _A    :				Any=None)   ->     str:
					__snake_case      :							Dict						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				List[str]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Tuple						=			[self.sep_token_id]
					__snake_case      :							Optional[Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



				def  _lowercase					(self    :				List[Any]     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							List[Any]						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)






				def  _lowercase					(self    :				int     ,  _A    :				Optional[int]     ,  _A    :				Tuple=None     ,  _A    :				Tuple=None     ,  _A    :				Dict=False     ,  **_A    :				Optional[int]     ,  )   ->     Optional[Any]:
					__snake_case      :							Optional[Any]						=			BertPreTokenizer()
					return super().save_pretrained(_A     ,  _A     ,  _A     ,  _A     ,  **_A)





