


'''simple docstring'''



import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase      :      Optional[int]            =					get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")






@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__(       snake_case_						,    unittest.TestCase	):
    __magic_name__    :				Any  		=	GPTSwaTokenizer
    __magic_name__    :				List[str]  		=	False
    __magic_name__    :				Any  		=	True
    __magic_name__    :				Union[str, Any]  		=	False




    def 					a__(		self      :			Optional[int]				)->					Dict:

          """simple docstring"""




          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase					    =     GPTSwaTokenizer(_A				,   eos_token='''<unk>'''				,   bos_token='''<unk>'''				,   pad_token='''<unk>'''				)

          tokenizer.save_pretrained(self.tmpdirname				)




    def 					a__(		self      :			int				,   lowerCAmelCase      :			Any				)->					str:

          """simple docstring"""




          UpperCAmelCase					    =     'This is a test'
          UpperCAmelCase					    =     'This is a test'
          return input_text, output_text




    def 					a__(		self      :			List[Any]				)->					Optional[Any]:

          """simple docstring"""




          UpperCAmelCase					    =     '<s>'
          UpperCAmelCase					    =     1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A				)				,   _A				)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A				)				,   _A				)




    def 					a__(		self      :			Dict				)->					List[Any]:

          """simple docstring"""




          UpperCAmelCase					    =     list(self.get_tokenizer().get_vocab().keys()				)

          self.assertEqual(vocab_keys[0]				,   '''<unk>'''				)
          self.assertEqual(vocab_keys[1]				,   '''<s>'''				)
          self.assertEqual(vocab_keys[-1]				,   '''j'''				)
          self.assertEqual(len(_A				)				,   2000				)




    def 					a__(		self      :			int				)->					int:

          """simple docstring"""




          self.assertEqual(self.get_tokenizer().vocab_size				,   2000				)




    def 					a__(		self      :			List[str]				)->					Optional[Any]:

          """simple docstring"""




          UpperCAmelCase					    =     GPTSwaTokenizer(_A				)

          UpperCAmelCase					    =     tokenizer.tokenize('''This is a test'''				)
          self.assertListEqual(_A				,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

          self.assertListEqual(tokenizer.convert_tokens_to_ids(_A				)				,   [465, 287, 265, 631, 842]				)

          UpperCAmelCase					    =     tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
          # fmt: off
          self.assertListEqual(
              _A				,   ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']				,   )
          # fmt: on

          UpperCAmelCase					    =     tokenizer.convert_tokens_to_ids(_A				)
          self.assertListEqual(
              _A				,   [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]				,   )

          UpperCAmelCase					    =     tokenizer.convert_ids_to_tokens(_A				)
          # fmt: off
          self.assertListEqual(
              _A				,   ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']				)
          # fmt: on




    def 					a__(		self      :			Union[str, Any]				)->					List[str]:

          """simple docstring"""




          UpperCAmelCase					    =     GPTSwaTokenizer(_A				)
          UpperCAmelCase					    =     ['This is a test', 'I was born in 92000, and this is falsé.']
          UpperCAmelCase					    =     [
              [465, 287, 265, 631, 842],
              [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
          ]

          # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
          for text, expected_ids in zip(_A				,   _A				):
                self.assertListEqual(tokenizer.encode_fast(_A				)				,   _A				)

          # Test that decode_fast returns the input text
          for text, token_ids in zip(_A				,   _A				):
                self.assertEqual(tokenizer.decode_fast(_A				)				,   _A				)





    @slow
    def 					a__(		self      :			Any				)->					str:

          """simple docstring"""




          UpperCAmelCase					    =     [
              '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
              'Hey there, how are you doing this fine day?',
              'This is a text with a trailing spaces followed by a dot     .',
              'Häj sväjs lillebrör! =)',
              'Det är inget fel på Mr. Cool',
          ]

          # fmt: off
          UpperCAmelCase					    =     {'input_ids': [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
          # fmt: on
          self.tokenizer_integration_test_util(
              expected_encoding=_A				,   model_name='''AI-Sweden/gpt-sw3-126m'''				,   sequences=_A				,   )









'''simple docstring'''



from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)





