"""simple docstring"""
import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 					lowerCAmelCase_      (     lowerCAmelCase       ,      unittest.TestCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      List[str]     					=  CodeGenTokenizer
				_lowerCAmelCase       :      Any     					=  CodeGenTokenizerFast
				_lowerCAmelCase       :      Union[str, Any]     					=  True
				_lowerCAmelCase       :      str     					=  {"""add_prefix_space""": True}
				_lowerCAmelCase       :      Tuple     					=  False

				def 						snake_case	(			self      ):






										"""simple docstring"""

										super().setUp()

										# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
										snake_case        =     [
										    'l',
										    'o',
										    'w',
										    'e',
										    'r',
										    's',
										    't',
										    'i',
										    'd',
										    'n',
										    '\u0120',
										    '\u0120l',
										    '\u0120n',
										    '\u0120lo',
										    '\u0120low',
										    'er',
										    '\u0120lowest',
										    '\u0120newer',
										    '\u0120wider',
										    '<unk>',
										    '<|endoftext|>',
										]
										snake_case        =     dict(zip(lowerCAmelCase						,      range(len(lowerCAmelCase      )      )      )      )
										snake_case        =     ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
										snake_case        =     {'unk_token': '<unk>'}

										snake_case        =     os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['vocab_file']      )
										snake_case        =     os.path.join(self.tmpdirname						,      VOCAB_FILES_NAMES['merges_file']      )
										with open(self.vocab_file						,      'w'						,      encoding='utf-8'      ) as fp:
																fp.write(json.dumps(lowerCAmelCase      ) + '\n'      )
										with open(self.merges_file						,      'w'						,      encoding='utf-8'      ) as fp:
																fp.write('\n'.join(lowerCAmelCase      )      )

				def 						snake_case	(			self						,      **lowerCAmelCase      ):






										"""simple docstring"""

										kwargs.update(self.special_tokens_map      )
										return CodeGenTokenizer.from_pretrained(self.tmpdirname						,      **lowerCAmelCase      )

				def 						snake_case	(			self						,      **lowerCAmelCase      ):






										"""simple docstring"""

										kwargs.update(self.special_tokens_map      )
										return CodeGenTokenizerFast.from_pretrained(self.tmpdirname						,      **lowerCAmelCase      )

				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     'lower newer'
										snake_case        =     'lower newer'
										return input_text, output_text

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     CodeGenTokenizer(self.vocab_file						,      self.merges_file						,      **self.special_tokens_map      )
										snake_case        =     'lower newer'
										snake_case        =     ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
										snake_case        =     tokenizer.tokenize(lowerCAmelCase						,      add_prefix_space=lowerCAmelCase      )
										self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

										snake_case        =     tokens + [tokenizer.unk_token]
										snake_case        =     [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase      )						,      lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										if not self.test_rust_tokenizer:
																return

										snake_case        =     self.get_tokenizer()
										snake_case        =     self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase      )

										snake_case        =     'lower newer'

										# Testing tokenization
										snake_case        =     tokenizer.tokenize(lowerCAmelCase						,      add_prefix_space=lowerCAmelCase      )
										snake_case        =     rust_tokenizer.tokenize(lowerCAmelCase      )
										self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

										# Testing conversion to ids without special tokens
										snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase						,      add_prefix_space=lowerCAmelCase      )
										snake_case        =     rust_tokenizer.encode(lowerCAmelCase						,      add_special_tokens=lowerCAmelCase      )
										self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

										# Testing conversion to ids with special tokens
										snake_case        =     self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase      )
										snake_case        =     tokenizer.encode(lowerCAmelCase						,      add_prefix_space=lowerCAmelCase      )
										snake_case        =     rust_tokenizer.encode(lowerCAmelCase      )
										self.assertListEqual(lowerCAmelCase						,      lowerCAmelCase      )

										# Testing the unknown token
										snake_case        =     tokens + [rust_tokenizer.unk_token]
										snake_case        =     [14, 15, 10, 9, 3, 2, 15, 19]
										self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase      )						,      lowerCAmelCase      )

				def 						snake_case	(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										pass

				def 						snake_case	(			self						,      lowerCAmelCase=15      ):






										"""simple docstring"""

										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
																						snake_case        =     self.rust_tokenizer_class.from_pretrained(lowerCAmelCase						,      **lowerCAmelCase      )

																						# Simple input
																						snake_case        =     'This is a simple input'
																						snake_case        =     ['This is a simple input 1', 'This is a simple input 2']
																						snake_case        =     ('This is a simple input', 'This is a pair')
																						snake_case        =     [
																						    ('This is a simple input 1', 'This is a simple input 2'),
																						    ('This is a simple pair 1', 'This is a simple pair 2'),
																						]

																						# Simple input tests
																						self.assertRaises(lowerCAmelCase						,      tokenizer_r.encode						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'      )

																						# Simple input
																						self.assertRaises(lowerCAmelCase						,      tokenizer_r.encode_plus						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'      )

																						# Simple input
																						self.assertRaises(
																						    lowerCAmelCase						,      tokenizer_r.batch_encode_plus						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'						,      )

																						# Pair input
																						self.assertRaises(lowerCAmelCase						,      tokenizer_r.encode						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'      )

																						# Pair input
																						self.assertRaises(lowerCAmelCase						,      tokenizer_r.encode_plus						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'      )

																						# Pair input
																						self.assertRaises(
																						    lowerCAmelCase						,      tokenizer_r.batch_encode_plus						,      lowerCAmelCase						,      max_length=lowerCAmelCase						,      padding='max_length'						,      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     CodeGenTokenizer.from_pretrained(self.tmpdirname						,      pad_token='<pad>'      )

										# Simple input
										snake_case        =     'This is a simple input'
										snake_case        =     ['This is a simple input looooooooong', 'This is a simple input']
										snake_case        =     ('This is a simple input', 'This is a pair')
										snake_case        =     [
										    ('This is a simple input loooooong', 'This is a simple input'),
										    ('This is a simple pair loooooong', 'This is a simple pair'),
										]

										snake_case        =     tokenizer.pad_token_id

										snake_case        =     tokenizer(lowerCAmelCase						,      padding='max_length'						,      max_length=30						,      return_tensors='np'      )
										snake_case        =     tokenizer(lowerCAmelCase						,      padding=lowerCAmelCase						,      truncate=lowerCAmelCase						,      return_tensors='np'      )
										snake_case        =     tokenizer(*lowerCAmelCase						,      padding='max_length'						,      max_length=60						,      return_tensors='np'      )
										snake_case        =     tokenizer(lowerCAmelCase						,      padding=lowerCAmelCase						,      truncate=lowerCAmelCase						,      return_tensors='np'      )

										# s
										# test single string max_length padding
										self.assertEqual(out_s['input_ids'].shape[-1]						,      30      )
										self.assertTrue(pad_token_id in out_s['input_ids']      )
										self.assertTrue(0 in out_s['attention_mask']      )

										# s2
										# test automatic padding
										self.assertEqual(out_sa['input_ids'].shape[-1]						,      33      )
										# long slice doesn't have padding
										self.assertFalse(pad_token_id in out_sa['input_ids'][0]      )
										self.assertFalse(0 in out_sa['attention_mask'][0]      )
										# short slice does have padding
										self.assertTrue(pad_token_id in out_sa['input_ids'][1]      )
										self.assertTrue(0 in out_sa['attention_mask'][1]      )

										# p
										# test single pair max_length padding
										self.assertEqual(out_p['input_ids'].shape[-1]						,      60      )
										self.assertTrue(pad_token_id in out_p['input_ids']      )
										self.assertTrue(0 in out_p['attention_mask']      )

										# p2
										# test automatic padding pair
										self.assertEqual(out_pa['input_ids'].shape[-1]						,      52      )
										# long slice pair doesn't have padding
										self.assertFalse(pad_token_id in out_pa['input_ids'][0]      )
										self.assertFalse(0 in out_pa['attention_mask'][0]      )
										# short slice pair does have padding
										self.assertTrue(pad_token_id in out_pa['input_ids'][1]      )
										self.assertTrue(0 in out_pa['attention_mask'][1]      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     '$$$'
										snake_case        =     CodeGenTokenizer.from_pretrained(self.tmpdirname						,      bos_token=lowerCAmelCase						,      add_bos_token=lowerCAmelCase      )

										snake_case        =     'This is a simple input'
										snake_case        =     ['This is a simple input 1', 'This is a simple input 2']

										snake_case        =     tokenizer.bos_token_id

										snake_case        =     tokenizer(lowerCAmelCase      )
										snake_case        =     tokenizer(lowerCAmelCase      )

										self.assertEqual(out_s.input_ids[0]						,      lowerCAmelCase      )
										self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

										snake_case        =     tokenizer.decode(out_s.input_ids      )
										snake_case        =     tokenizer.batch_decode(out_sa.input_ids      )

										self.assertEqual(decode_s.split()[0]						,      lowerCAmelCase      )
										self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )

				@slow
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'      )

										snake_case        =     '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
										snake_case        =     '\nif len_a > len_b:      result = a\nelse:      result = b'

										snake_case        =     tokenizer.encode(lowerCAmelCase      )
										snake_case        =     ['^#', re.escape('<|endoftext|>'      ), '^\'\'\'', '^"""', '\n\n\n']
										snake_case        =     tokenizer.decode(lowerCAmelCase						,      truncate_before_pattern=lowerCAmelCase      )
										self.assertEqual(lowerCAmelCase						,      lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass



"""simple docstring"""
from __future__ import annotations

import unittest

from transformers import DebertaVaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFDebertaVaForMaskedLM,
					    TFDebertaVaForQuestionAnswering,
					    TFDebertaVaForSequenceClassification,
					    TFDebertaVaForTokenClassification,
					    TFDebertaVaModel,
					)



class 					lowerCAmelCase_      :



				"""simple docstring"""

				def __init__(			self						,      lowerCAmelCase						,      lowerCAmelCase=13						,      lowerCAmelCase=7						,      lowerCAmelCase=True						,      lowerCAmelCase=True						,      lowerCAmelCase=True						,      lowerCAmelCase=True						,      lowerCAmelCase=99						,      lowerCAmelCase=32						,      lowerCAmelCase=2						,      lowerCAmelCase=4						,      lowerCAmelCase=37						,      lowerCAmelCase="gelu"						,      lowerCAmelCase=0.1						,      lowerCAmelCase=0.1						,      lowerCAmelCase=5_12						,      lowerCAmelCase=16						,      lowerCAmelCase=2						,      lowerCAmelCase=0.02						,      lowerCAmelCase=False						,      lowerCAmelCase=True						,      lowerCAmelCase="None"						,      lowerCAmelCase=3						,      lowerCAmelCase=4						,      lowerCAmelCase=None						,      ):






										"""simple docstring"""

										snake_case        =     parent
										snake_case        =     batch_size
										snake_case        =     seq_length
										snake_case        =     is_training
										snake_case        =     use_input_mask
										snake_case        =     use_token_type_ids
										snake_case        =     use_labels
										snake_case        =     vocab_size
										snake_case        =     hidden_size
										snake_case        =     num_hidden_layers
										snake_case        =     num_attention_heads
										snake_case        =     intermediate_size
										snake_case        =     hidden_act
										snake_case        =     hidden_dropout_prob
										snake_case        =     attention_probs_dropout_prob
										snake_case        =     max_position_embeddings
										snake_case        =     type_vocab_size
										snake_case        =     type_sequence_label_size
										snake_case        =     initializer_range
										snake_case        =     num_labels
										snake_case        =     num_choices
										snake_case        =     relative_attention
										snake_case        =     position_biased_input
										snake_case        =     pos_att_type
										snake_case        =     scope

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     ids_tensor([self.batch_size, self.seq_length]						,      self.vocab_size      )

										snake_case        =     None
										if self.use_input_mask:
																snake_case        =     random_attention_mask([self.batch_size, self.seq_length]      )

										snake_case        =     None
										if self.use_token_type_ids:
																snake_case        =     ids_tensor([self.batch_size, self.seq_length]						,      self.type_vocab_size      )

										snake_case        =     None
										snake_case        =     None
										snake_case        =     None
										if self.use_labels:
																snake_case        =     ids_tensor([self.batch_size]						,      self.type_sequence_label_size      )
																snake_case        =     ids_tensor([self.batch_size, self.seq_length]						,      self.num_labels      )

										snake_case        =     DebertaVaConfig(
										    vocab_size=self.vocab_size						,      hidden_size=self.hidden_size						,      num_hidden_layers=self.num_hidden_layers						,      num_attention_heads=self.num_attention_heads						,      intermediate_size=self.intermediate_size						,      hidden_act=self.hidden_act						,      hidden_dropout_prob=self.hidden_dropout_prob						,      attention_probs_dropout_prob=self.attention_probs_dropout_prob						,      max_position_embeddings=self.max_position_embeddings						,      type_vocab_size=self.type_vocab_size						,      relative_attention=self.relative_attention						,      position_biased_input=self.position_biased_input						,      initializer_range=self.initializer_range						,      return_dict=lowerCAmelCase						,      )

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaModel(config=lowerCAmelCase      )
										snake_case        =     {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

										snake_case        =     [input_ids, input_mask]
										snake_case        =     model(lowerCAmelCase      )

										snake_case        =     model(lowerCAmelCase      )

										self.parent.assertEqual(result.last_hidden_state.shape						,      (self.batch_size, self.seq_length, self.hidden_size)      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaForMaskedLM(config=lowerCAmelCase      )
										snake_case        =     {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}
										snake_case        =     model(lowerCAmelCase      )
										self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.vocab_size)      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     self.num_labels
										snake_case        =     TFDebertaVaForSequenceClassification(config=lowerCAmelCase      )
										snake_case        =     {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}

										snake_case        =     model(lowerCAmelCase      )
										self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.num_labels)      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     self.num_labels
										snake_case        =     TFDebertaVaForTokenClassification(config=lowerCAmelCase      )
										snake_case        =     {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}
										snake_case        =     model(lowerCAmelCase      )
										self.parent.assertEqual(result.logits.shape						,      (self.batch_size, self.seq_length, self.num_labels)      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaForQuestionAnswering(config=lowerCAmelCase      )
										snake_case        =     {
										    'input_ids': input_ids,
										    'attention_mask': input_mask,
										    'token_type_ids': token_type_ids,
										}

										snake_case        =     model(lowerCAmelCase      )
										self.parent.assertEqual(result.start_logits.shape						,      (self.batch_size, self.seq_length)      )
										self.parent.assertEqual(result.end_logits.shape						,      (self.batch_size, self.seq_length)      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.prepare_config_and_inputs()
										(
										    (
										    snake_case
										)   ,(
										    snake_case
										)   ,(
										    snake_case
										)   ,(
										    snake_case
										)   ,(
										    snake_case
										)   ,(
										    snake_case
										)   ,(
										    snake_case
										)   ,
										)        =     config_and_inputs
										snake_case        =     {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
										return config, inputs_dict





@require_tf
class 					lowerCAmelCase_      (     lowerCAmelCase       ,      lowerCAmelCase       ,      unittest.TestCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[int]     					=  (
				    (
				        TFDebertaVaModel,
				        TFDebertaVaForMaskedLM,
				        TFDebertaVaForQuestionAnswering,
				        TFDebertaVaForSequenceClassification,
				        TFDebertaVaForTokenClassification,
				    )
				    if is_tf_available()
				    else ()
				)
				_lowerCAmelCase       :      Any     					=  (
				    {
				        """feature-extraction""": TFDebertaVaModel,
				        """fill-mask""": TFDebertaVaForMaskedLM,
				        """question-answering""": TFDebertaVaForQuestionAnswering,
				        """text-classification""": TFDebertaVaForSequenceClassification,
				        """token-classification""": TFDebertaVaForTokenClassification,
				        """zero-shot""": TFDebertaVaForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)

				_lowerCAmelCase       :      Optional[Any]     					=  False
				_lowerCAmelCase       :      List[Any]     					=  False

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaModelTester(self      )
										snake_case        =     ConfigTester(self						,      config_class=lowerCAmelCase						,      hidden_size=37      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										self.config_tester.run_common_tests()

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase      )

				@slow
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge'      )
										self.assertIsNotNone(lowerCAmelCase      )





@require_tf
class 					lowerCAmelCase_      (     unittest.TestCase   ):



				"""simple docstring"""

				@unittest.skip(reason='Model not available yet'      )
				def 						snake_case	(			self      ):






										"""simple docstring"""

										pass

				@slow
				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     TFDebertaVaModel.from_pretrained('kamalkraj/deberta-v2-xlarge'      )
										snake_case        =     tf.constant([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]      )
										snake_case        =     tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
										snake_case        =     model(lowerCAmelCase						,      attention_mask=lowerCAmelCase      )[0]

										snake_case        =     tf.constant(
										    [[[0.23_56, 0.19_48, 0.03_69], [-0.10_63, 0.35_86, -0.51_52], [-0.63_99, -0.02_59, -0.25_25]]]      )
										tf.debugging.assert_near(output[:, 1:4, 1:4]						,      lowerCAmelCase						,      atol=1E-4      )


