






"""simple docstring"""




import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    DistilBertForMaskedLM,
		    DistilBertForMultipleChoice,
		    DistilBertForQuestionAnswering,
		    DistilBertForSequenceClassification,
		    DistilBertForTokenClassification,
		    DistilBertModel,
		)



class 				__UpperCAmelCase		(UpperCAmelCase__   ):







			def __init__(     self:  int				,     UpperCAmelCase_:  Tuple				,     UpperCAmelCase_:  Optional[Any]=13				,     UpperCAmelCase_:  Tuple=7				,     UpperCAmelCase_:  Dict=True				,     UpperCAmelCase_:  str=True				,     UpperCAmelCase_:  List[Any]=False				,     UpperCAmelCase_:  Dict=True				,     UpperCAmelCase_:  Optional[int]=99				,     UpperCAmelCase_:  int=32				,     UpperCAmelCase_:  Tuple=5				,     UpperCAmelCase_:  Union[str, Any]=4				,     UpperCAmelCase_:  Union[str, Any]=37				,     UpperCAmelCase_:  Optional[int]="gelu"				,     UpperCAmelCase_:  List[str]=0.1				,     UpperCAmelCase_:  List[Any]=0.1				,     UpperCAmelCase_:  List[Any]=512				,     UpperCAmelCase_:  Union[str, Any]=16				,     UpperCAmelCase_:  Optional[Any]=2				,     UpperCAmelCase_:  List[Any]=0.02				,     UpperCAmelCase_:  Any=3				,     UpperCAmelCase_:  Any=4				,     UpperCAmelCase_:  Dict=None				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  parent
									_SCREAMING_SNAKE_CASE     				=  batch_size
									_SCREAMING_SNAKE_CASE     				=  seq_length
									_SCREAMING_SNAKE_CASE     				=  is_training
									_SCREAMING_SNAKE_CASE     				=  use_input_mask
									_SCREAMING_SNAKE_CASE     				=  use_token_type_ids
									_SCREAMING_SNAKE_CASE     				=  use_labels
									_SCREAMING_SNAKE_CASE     				=  vocab_size
									_SCREAMING_SNAKE_CASE     				=  hidden_size
									_SCREAMING_SNAKE_CASE     				=  num_hidden_layers
									_SCREAMING_SNAKE_CASE     				=  num_attention_heads
									_SCREAMING_SNAKE_CASE     				=  intermediate_size
									_SCREAMING_SNAKE_CASE     				=  hidden_act
									_SCREAMING_SNAKE_CASE     				=  hidden_dropout_prob
									_SCREAMING_SNAKE_CASE     				=  attention_probs_dropout_prob
									_SCREAMING_SNAKE_CASE     				=  max_position_embeddings
									_SCREAMING_SNAKE_CASE     				=  type_vocab_size
									_SCREAMING_SNAKE_CASE     				=  type_sequence_label_size
									_SCREAMING_SNAKE_CASE     				=  initializer_range
									_SCREAMING_SNAKE_CASE     				=  num_labels
									_SCREAMING_SNAKE_CASE     				=  num_choices
									_SCREAMING_SNAKE_CASE     				=  scope







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size					)

									_SCREAMING_SNAKE_CASE     				=  None
									if self.use_input_mask:
															_SCREAMING_SNAKE_CASE     				=  random_attention_mask([self.batch_size, self.seq_length]					)

									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  None
									if self.use_labels:
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size]				,     self.type_sequence_label_size					)
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels					)
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size]				,     self.num_choices					)

									_SCREAMING_SNAKE_CASE     				=  self.get_config()

									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







			def 							UpperCamelCase				(     self:  Dict					):







									'''simple docstring'''




									return DistilBertConfig(
									    vocab_size=self.vocab_size				,     dim=self.hidden_size				,     n_layers=self.num_hidden_layers				,     n_heads=self.num_attention_heads				,     hidden_dim=self.intermediate_size				,     hidden_act=self.hidden_act				,     dropout=self.hidden_dropout_prob				,     attention_dropout=self.attention_probs_dropout_prob				,     max_position_embeddings=self.max_position_embeddings				,     initializer_range=self.initializer_range				,     )







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  DistilBertModel(config=_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE				,     _SCREAMING_SNAKE_CASE					)
									_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE					)
									self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size)					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  DistilBertForMaskedLM(config=_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE				,     labels=_SCREAMING_SNAKE_CASE					)
									self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.vocab_size)					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  DistilBertForQuestionAnswering(config=_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(
									    _SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE				,     start_positions=_SCREAMING_SNAKE_CASE				,     end_positions=_SCREAMING_SNAKE_CASE					)
									self.parent.assertEqual(result.start_logits.shape				,     (self.batch_size, self.seq_length)					)
									self.parent.assertEqual(result.end_logits.shape				,     (self.batch_size, self.seq_length)					)







			def 							UpperCamelCase				(     self:  Any				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.num_labels
									_SCREAMING_SNAKE_CASE     				=  DistilBertForSequenceClassification(_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE				,     labels=_SCREAMING_SNAKE_CASE					)
									self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_labels)					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.num_labels
									_SCREAMING_SNAKE_CASE     				=  DistilBertForTokenClassification(config=_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()

									_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE				,     labels=_SCREAMING_SNAKE_CASE					)
									self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.num_labels)					)







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Tuple				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.num_choices
									_SCREAMING_SNAKE_CASE     				=  DistilBertForMultipleChoice(config=_SCREAMING_SNAKE_CASE					)
									model.to(_SCREAMING_SNAKE_CASE					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  input_ids.unsqueeze(1					).expand(-1				,     self.num_choices				,     -1					).contiguous()
									_SCREAMING_SNAKE_CASE     				=  input_mask.unsqueeze(1					).expand(-1				,     self.num_choices				,     -1					).contiguous()
									_SCREAMING_SNAKE_CASE     				=  model(
									    _SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE				,     labels=_SCREAMING_SNAKE_CASE				,     )
									self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.num_choices)					)







			def 							UpperCamelCase				(     self:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.prepare_config_and_inputs()
									(_SCREAMING_SNAKE_CASE)     				=  config_and_inputs
									_SCREAMING_SNAKE_CASE     				=  {"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict



@require_torch
class 				__UpperCAmelCase		(UpperCAmelCase__	,UpperCAmelCase__	,unittest.TestCase   ):
			__snake_case    :				List[str]									=							(
			    (
			        DistilBertModel,
			        DistilBertForMaskedLM,
			        DistilBertForMultipleChoice,
			        DistilBertForQuestionAnswering,
			        DistilBertForSequenceClassification,
			        DistilBertForTokenClassification,
			    )
			    if is_torch_available()
			    else None
			)
			__snake_case    :				Optional[int]									=							(
			    {
			        'feature-extraction': DistilBertModel,
			        'fill-mask': DistilBertForMaskedLM,
			        'question-answering': DistilBertForQuestionAnswering,
			        'text-classification': DistilBertForSequenceClassification,
			        'token-classification': DistilBertForTokenClassification,
			        'zero-shot': DistilBertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__snake_case    :				Dict									=							True
			__snake_case    :				Optional[Any]									=							True
			__snake_case    :				int									=							True
			__snake_case    :				str									=							True







			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  DistilBertModelTester(self					)
									_SCREAMING_SNAKE_CASE     				=  ConfigTester(self				,     config_class=_SCREAMING_SNAKE_CASE				,     dim=37					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									self.config_tester.run_common_tests()







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_model(*_SCREAMING_SNAKE_CASE					)







			def 							UpperCamelCase				(     self:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_for_masked_lm(*_SCREAMING_SNAKE_CASE					)







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_for_question_answering(*_SCREAMING_SNAKE_CASE					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_for_sequence_classification(*_SCREAMING_SNAKE_CASE					)







			def 							UpperCamelCase				(     self:  Dict					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_for_token_classification(*_SCREAMING_SNAKE_CASE					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_distilbert_for_multiple_choice(*_SCREAMING_SNAKE_CASE					)







			@slow
			def 							UpperCamelCase				(     self:  Tuple					):







									'''simple docstring'''




									for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															_SCREAMING_SNAKE_CASE     				=  DistilBertModel.from_pretrained(_SCREAMING_SNAKE_CASE					)
															self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)







			@slow
			@require_torch_gpu
			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()
									for model_class in self.all_model_classes:
															# BertForMultipleChoice behaves incorrectly in JIT environments.
															if model_class == DistilBertForMultipleChoice:
																					return

															_SCREAMING_SNAKE_CASE     				=  True
															_SCREAMING_SNAKE_CASE     				=  model_class(config=_SCREAMING_SNAKE_CASE					)

															_SCREAMING_SNAKE_CASE     				=  self._prepare_for_class(_SCREAMING_SNAKE_CASE				,     _SCREAMING_SNAKE_CASE					)
															_SCREAMING_SNAKE_CASE     				=  torch.jit.trace(
															    _SCREAMING_SNAKE_CASE				,     (inputs_dict["""input_ids"""].to("""cpu"""					), inputs_dict["""attention_mask"""].to("""cpu"""					))					)

															with tempfile.TemporaryDirectory() as tmp:
																					torch.jit.save(_SCREAMING_SNAKE_CASE				,     os.path.join(_SCREAMING_SNAKE_CASE				,     """traced_model.pt"""					)					)
																					_SCREAMING_SNAKE_CASE     				=  torch.jit.load(os.path.join(_SCREAMING_SNAKE_CASE				,     """traced_model.pt"""					)				,     map_location=_SCREAMING_SNAKE_CASE					)
																					loaded(inputs_dict["""input_ids"""].to(_SCREAMING_SNAKE_CASE					)				,     inputs_dict["""attention_mask"""].to(_SCREAMING_SNAKE_CASE					)					)



@require_torch
class 				__UpperCAmelCase		(unittest.TestCase   ):







			@slow
			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  DistilBertModel.from_pretrained("""distilbert-base-uncased"""					)
									_SCREAMING_SNAKE_CASE     				=  torch.tensor([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]					)
									_SCREAMING_SNAKE_CASE     				=  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]					)
									with torch.no_grad():
															_SCREAMING_SNAKE_CASE     				=  model(_SCREAMING_SNAKE_CASE				,     attention_mask=_SCREAMING_SNAKE_CASE					)[0]
									_SCREAMING_SNAKE_CASE     				=  torch.Size((1, 11, 768)					)
									self.assertEqual(output.shape				,     _SCREAMING_SNAKE_CASE					)
									_SCREAMING_SNAKE_CASE     				=  torch.tensor(
									    [[[-0.16_39, 0.32_99, 0.16_48], [-0.17_46, 0.32_89, 0.17_10], [-0.18_84, 0.33_57, 0.18_10]]]					)

									self.assertTrue(torch.allclose(output[:, 1:4, 1:4]				,     _SCREAMING_SNAKE_CASE				,     atol=1E-4					)					)









import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_camembert import CamembertTokenizer
else:
		UpperCamelCase			    =     None


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''camembert-base''': '''https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_file''': {
        '''camembert-base''': '''https://huggingface.co/camembert-base/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase			    =     {
    '''camembert-base''': 512,
}

UpperCamelCase			    =     '''▁'''



class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			__snake_case    :				int									=							VOCAB_FILES_NAMES
			__snake_case    :				Any									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				Optional[Any]									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				Dict									=							["input_ids", "attention_mask"]
			__snake_case    :				Tuple									=							CamembertTokenizer







			def __init__(     self:  List[Any]				,     UpperCAmelCase_:  Optional[int]=None				,     UpperCAmelCase_:  Tuple=None				,     UpperCAmelCase_:  str="<s>"				,     UpperCAmelCase_:  List[str]="</s>"				,     UpperCAmelCase_:  Dict="</s>"				,     UpperCAmelCase_:  List[Any]="<s>"				,     UpperCAmelCase_:  Dict="<unk>"				,     UpperCAmelCase_:  Any="<pad>"				,     UpperCAmelCase_:  Tuple="<mask>"				,     UpperCAmelCase_:  str=["<s>NOTUSED", "</s>NOTUSED"]				,     **UpperCAmelCase_:  Optional[Any]				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  AddedToken(UpperCAmelCase_				,     lstrip=UpperCAmelCase_				,     rstrip=UpperCAmelCase_					) if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					) else mask_token

									super().__init__(
									    UpperCAmelCase_				,     tokenizer_file=UpperCAmelCase_				,     bos_token=UpperCAmelCase_				,     eos_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     mask_token=UpperCAmelCase_				,     additional_special_tokens=UpperCAmelCase_				,     **UpperCAmelCase_				,     )

									_SCREAMING_SNAKE_CASE     				=  vocab_file
									_SCREAMING_SNAKE_CASE     				=  False if not self.vocab_file else True







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""					)

									if not os.path.isdir(UpperCAmelCase_					):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
															return
									_SCREAMING_SNAKE_CASE     				=  os.path.join(
									    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCAmelCase_					):
															copyfile(self.vocab_file				,     UpperCAmelCase_					)

									return (out_vocab_file,)

