






from __future__ import annotations

import unittest

from transformers import DistilBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers.models.distilbert.modeling_tf_distilbert import (
		    TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFDistilBertForMaskedLM,
		    TFDistilBertForMultipleChoice,
		    TFDistilBertForQuestionAnswering,
		    TFDistilBertForSequenceClassification,
		    TFDistilBertForTokenClassification,
		    TFDistilBertModel,
		)





class 	snake_case__ :
	def __init__(    self      ,       lowerCAmelCase__      ,       )							->				Optional[Any]:
							__magic_name__      :       Dict       						= parent
							__magic_name__      :       Optional[int]       						= 13
							__magic_name__      :       Any       						= 7
							__magic_name__      :       str       						= True
							__magic_name__      :       List[Any]       						= True
							__magic_name__      :       Tuple       						= False
							__magic_name__      :       Dict       						= True
							__magic_name__      :       Any       						= 99
							__magic_name__      :       int       						= 32
							__magic_name__      :       Optional[int]       						= 2
							__magic_name__      :       Tuple       						= 4
							__magic_name__      :       Any       						= 37
							__magic_name__      :       Tuple       						= """gelu"""
							__magic_name__      :       List[Any]       						= 0.1
							__magic_name__      :       str       						= 0.1
							__magic_name__      :       Optional[int]       						= 5_12
							__magic_name__      :       Optional[Any]       						= 16
							__magic_name__      :       Union[str, Any]       						= 2
							__magic_name__      :       Any       						= 0.0_2
							__magic_name__      :       Tuple       						= 3
							__magic_name__      :       Union[str, Any]       						= 4
							__magic_name__      :       List[Any]       						= None
	def       __magic_name__    (    self			)							->				Any:
							__magic_name__      :       Optional[Any]       						= ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size			)

							__magic_name__      :       int       						= None
							if self.use_input_mask:
													__magic_name__      :       Tuple       						= random_attention_mask([self.batch_size, self.seq_length]			)

							__magic_name__      :       List[Any]       						= None
							__magic_name__      :       Any       						= None
							__magic_name__      :       Optional[Any]       						= None
							if self.use_labels:
													__magic_name__      :       str       						= ids_tensor([self.batch_size]      ,       self.type_sequence_label_size			)
													__magic_name__      :       Optional[int]       						= ids_tensor([self.batch_size, self.seq_length]      ,       self.num_labels			)
													__magic_name__      :       Dict       						= ids_tensor([self.batch_size]      ,       self.num_choices			)

							__magic_name__      :       Any       						= DistilBertConfig(
							    vocab_size=self.vocab_size      ,       dim=self.hidden_size      ,       n_layers=self.num_hidden_layers      ,       n_heads=self.num_attention_heads      ,       hidden_dim=self.intermediate_size      ,       hidden_act=self.hidden_act      ,       dropout=self.hidden_dropout_prob      ,       attention_dropout=self.attention_probs_dropout_prob      ,       max_position_embeddings=self.max_position_embeddings      ,       initializer_range=self.initializer_range      ,       )

							return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				Optional[int]:
							__magic_name__      :       str       						= TFDistilBertModel(config=lowerCAmelCase__			)
							__magic_name__      :       Optional[Any]       						= {"""input_ids""": input_ids, """attention_mask""": input_mask}

							__magic_name__      :       Dict       						= model(lowerCAmelCase__			)

							__magic_name__      :       Dict       						= [input_ids, input_mask]

							__magic_name__      :       Dict       						= model(lowerCAmelCase__			)

							self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.hidden_size)			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				List[str]:
							__magic_name__      :       Optional[Any]       						= TFDistilBertForMaskedLM(config=lowerCAmelCase__			)
							__magic_name__      :       str       						= {"""input_ids""": input_ids, """attention_mask""": input_mask}
							__magic_name__      :       List[Any]       						= model(lowerCAmelCase__			)
							self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.vocab_size)			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				Dict:
							__magic_name__      :       Dict       						= TFDistilBertForQuestionAnswering(config=lowerCAmelCase__			)
							__magic_name__      :       List[Any]       						= {
							    """input_ids""": input_ids,
							    """attention_mask""": input_mask,
							}
							__magic_name__      :       Optional[Any]       						= model(lowerCAmelCase__			)
							self.parent.assertEqual(result.start_logits.shape      ,       (self.batch_size, self.seq_length)			)
							self.parent.assertEqual(result.end_logits.shape      ,       (self.batch_size, self.seq_length)			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				List[Any]:
							__magic_name__      :       Optional[int]       						= self.num_labels
							__magic_name__      :       Optional[int]       						= TFDistilBertForSequenceClassification(lowerCAmelCase__			)
							__magic_name__      :       Optional[Any]       						= {"""input_ids""": input_ids, """attention_mask""": input_mask}
							__magic_name__      :       Union[str, Any]       						= model(lowerCAmelCase__			)
							self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_labels)			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				str:
							__magic_name__      :       Dict       						= self.num_choices
							__magic_name__      :       Dict       						= TFDistilBertForMultipleChoice(lowerCAmelCase__			)
							__magic_name__      :       str       						= tf.tile(tf.expand_dims(lowerCAmelCase__      ,       1			)      ,       (1, self.num_choices, 1)			)
							__magic_name__      :       List[Any]       						= tf.tile(tf.expand_dims(lowerCAmelCase__      ,       1			)      ,       (1, self.num_choices, 1)			)
							__magic_name__      :       Optional[int]       						= {
							    """input_ids""": multiple_choice_inputs_ids,
							    """attention_mask""": multiple_choice_input_mask,
							}
							__magic_name__      :       Dict       						= model(lowerCAmelCase__			)
							self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_choices)			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__			)							->				Tuple:
							__magic_name__      :       List[Any]       						= self.num_labels
							__magic_name__      :       Optional[Any]       						= TFDistilBertForTokenClassification(lowerCAmelCase__			)
							__magic_name__      :       Any       						= {"""input_ids""": input_ids, """attention_mask""": input_mask}
							__magic_name__      :       List[str]       						= model(lowerCAmelCase__			)
							self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.num_labels)			)







	def       __magic_name__    (    self			)							->				Any:
							__magic_name__      :       Tuple       						= self.prepare_config_and_inputs()
							((__magic_name__)						,(__magic_name__)						,(__magic_name__)						,(__magic_name__)						,(__magic_name__)						,(__magic_name__))      :       str       						= config_and_inputs
							__magic_name__      :       str       						= {"""input_ids""": input_ids, """attention_mask""": input_mask}
							return config, inputs_dict





@require_tf
class 	snake_case__ (							_lowerCAmelCase						,  _lowerCAmelCase						,  unittest.TestCase				):
	lowercase__						:					Optional[int]											=   (
	    (
	        TFDistilBertModel,
	        TFDistilBertForMaskedLM,
	        TFDistilBertForQuestionAnswering,
	        TFDistilBertForSequenceClassification,
	        TFDistilBertForTokenClassification,
	        TFDistilBertForMultipleChoice,
	    )
	    if is_tf_available()
	    else None
	)
	lowercase__						:					Union[str, Any]											=   (
	    {
	        '''feature-extraction''': TFDistilBertModel,
	        '''fill-mask''': TFDistilBertForMaskedLM,
	        '''question-answering''': TFDistilBertForQuestionAnswering,
	        '''text-classification''': TFDistilBertForSequenceClassification,
	        '''token-classification''': TFDistilBertForTokenClassification,
	        '''zero-shot''': TFDistilBertForSequenceClassification,
	    }
	    if is_tf_available()
	    else {}
	)
	lowercase__						:					Any											=   False
	lowercase__						:					Optional[Any]											=   False
	def       __magic_name__    (    self			)							->				Any:
							__magic_name__      :       List[Any]       						= TFDistilBertModelTester(self			)
							__magic_name__      :       List[str]       						= ConfigTester(self      ,       config_class=lowerCAmelCase__      ,       dim=37			)
	def       __magic_name__    (    self			)							->				List[Any]:
							self.config_tester.run_common_tests()
	def       __magic_name__    (    self			)							->				Union[str, Any]:
							__magic_name__      :       Optional[int]       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_model(*lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Optional[Any]:
							__magic_name__      :       Union[str, Any]       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_masked_lm(*lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				List[str]:
							__magic_name__      :       Union[str, Any]       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_question_answering(*lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Tuple:
							__magic_name__      :       str       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_sequence_classification(*lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				List[str]:
							__magic_name__      :       Any       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_multiple_choice(*lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Dict:
							__magic_name__      :       Any       						= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_distilbert_for_token_classification(*lowerCAmelCase__			)







	@slow
	def       __magic_name__    (    self			)							->				Dict:
							for model_name in list(TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]			):
													__magic_name__      :       List[str]       						= TFDistilBertModel.from_pretrained(lowerCAmelCase__			)
													self.assertIsNotNone(lowerCAmelCase__			)





@require_tf
class 	snake_case__ (							unittest.TestCase				):
	@slow
	def       __magic_name__    (    self			)							->				Optional[Any]:
							__magic_name__      :       List[Any]       						= TFDistilBertModel.from_pretrained("""distilbert-base-uncased"""			)
							__magic_name__      :       str       						= tf.constant([[0, 1, 2, 3, 4, 5]]			)
							__magic_name__      :       Optional[Any]       						= model(lowerCAmelCase__			)[0]

							__magic_name__      :       Optional[int]       						= [1, 6, 7_68]
							self.assertEqual(output.shape      ,       lowerCAmelCase__			)

							__magic_name__      :       Optional[Any]       						= tf.constant(
							    [
							        [
							            [0.1_9_2_6_1_8_8_5, -0.1_3_7_3_2_9_5_5, 0.4_1_1_9_7_9_9],
							            [0.2_2_1_5_0_1_5_6, -0.0_7_4_2_2_6_6_1, 0.3_9_0_3_7_2_0_4],
							            [0.2_2_7_5_6_0_1_8, -0.0_8_9_6_4_1_4, 0.3_7_0_1_4_6_7],
							        ]
							    ]			)
							tf.debugging.assert_near(output[:, :3, :3]      ,       lowerCAmelCase__      ,       atol=1e-4			)








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_xlnet import XLNetTokenizer
else:
		__magic_name__:					Any							=     None


__magic_name__:					Dict							=     logging.get_logger(__name__)

__magic_name__:					Any							=     {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

__magic_name__:					str							=     {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json",
    },
}

__magic_name__:					Optional[Any]							=     {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

__magic_name__:					Optional[Any]							=     "▁"

# Segments (not really needed)
__magic_name__:					List[Any]							=     0
__magic_name__:					Dict							=     1
__magic_name__:					List[str]							=     2
__magic_name__:					List[Any]							=     3
__magic_name__:					Optional[int]							=     4





class 	snake_case__ (							_lowerCAmelCase				):
	lowercase__						:					Dict											=   VOCAB_FILES_NAMES
	lowercase__						:					Union[str, Any]											=   PRETRAINED_VOCAB_FILES_MAP
	lowercase__						:					Tuple											=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						:					List[str]											=   '''left'''
	lowercase__						:					List[str]											=   XLNetTokenizer
	def __init__(    self      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__=True      ,       lowerCAmelCase__=False      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="<unk>"      ,       lowerCAmelCase__="<sep>"      ,       lowerCAmelCase__="<pad>"      ,       lowerCAmelCase__="<cls>"      ,       lowerCAmelCase__="<mask>"      ,       lowerCAmelCase__=["<eop>", "<eod>"]      ,       **lowerCAmelCase__      ,       )							->				Tuple:
							# Mask token behave like a normal word, i.e. include the space before it
							__magic_name__      :       Optional[int]       						= AddedToken(lowerCAmelCase__      ,       lstrip=lowerCAmelCase__      ,       rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			) else mask_token

							super().__init__(
							    vocab_file=lowerCAmelCase__      ,       tokenizer_file=lowerCAmelCase__      ,       do_lower_case=lowerCAmelCase__      ,       remove_space=lowerCAmelCase__      ,       keep_accents=lowerCAmelCase__      ,       bos_token=lowerCAmelCase__      ,       eos_token=lowerCAmelCase__      ,       unk_token=lowerCAmelCase__      ,       sep_token=lowerCAmelCase__      ,       pad_token=lowerCAmelCase__      ,       cls_token=lowerCAmelCase__      ,       mask_token=lowerCAmelCase__      ,       additional_special_tokens=lowerCAmelCase__      ,       **lowerCAmelCase__      ,       )

							__magic_name__      :       List[str]       						= 3
							__magic_name__      :       str       						= do_lower_case
							__magic_name__      :       Union[str, Any]       						= remove_space
							__magic_name__      :       str       						= keep_accents
							__magic_name__      :       Tuple       						= vocab_file
							__magic_name__      :       List[Any]       						= False if not self.vocab_file else True
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       Any       						= [self.sep_token_id]
							__magic_name__      :       Any       						= [self.cls_token_id]
							if token_ids_a is None:
													return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       List[str]       						= [self.sep_token_id]
							__magic_name__      :       Optional[Any]       						= [2]

							if token_ids_a is None:
													return len(token_ids_a + sep			) * [0] + cls_segment_id
							return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id







	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				Tuple[str]:
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""			)

							if not os.path.isdir(lowerCAmelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
													return
							__magic_name__      :       List[str]       						= os.path.join(
							    lowerCAmelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCAmelCase__			):
													copyfile(self.vocab_file      ,       lowerCAmelCase__			)

							return (out_vocab_file,)
