"""simple docstring"""




import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_a				:		List[str]	   =       get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')
@require_sentencepiece
@require_tokenizers
class 						__A							(			a__   ,     unittest.TestCase		):
					_UpperCamelCase							:      Optional[Any]		 =		GPTSwaTokenizer
					_UpperCamelCase							:      Tuple		 =		False
					_UpperCamelCase							:      List[str]		 =		True
					_UpperCamelCase							:      List[Any]		 =		False






					def    __A  (		self  ):
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowerCAmelCase						:	Optional[Any]        =					GPTSwaTokenizer(lowerCAmelCase__    ,					eos_token="""<unk>"""    ,					bos_token="""<unk>"""    ,					pad_token="""<unk>"""  )

							tokenizer.save_pretrained(self.tmpdirname  )






					def    __A  (		self    ,					a__  ):
							_lowerCAmelCase						:	Any        =					"This is a test"
							_lowerCAmelCase						:	List[Any]        =					"This is a test"
							return input_text, output_text






					def    __A  (		self  ):
							_lowerCAmelCase						:	Optional[Any]        =					"<s>"
							_lowerCAmelCase						:	List[str]        =					1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__  )    ,					lowerCAmelCase__  )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__  )    ,					lowerCAmelCase__  )






					def    __A  (		self  ):
							_lowerCAmelCase						:	Optional[int]        =					list(self.get_tokenizer().get_vocab().keys()  )

							self.assertEqual(vocab_keys[0]    ,					"""<unk>"""  )
							self.assertEqual(vocab_keys[1]    ,					"""<s>"""  )
							self.assertEqual(vocab_keys[-1]    ,					"""j"""  )
							self.assertEqual(len(lowerCAmelCase__  )    ,					2000  )






					def    __A  (		self  ):
							self.assertEqual(self.get_tokenizer().vocab_size    ,					2000  )






					def    __A  (		self  ):
							_lowerCAmelCase						:	str        =					GPTSwaTokenizer(lowerCAmelCase__  )

							_lowerCAmelCase						:	Dict        =					tokenizer.tokenize("""This is a test"""  )
							self.assertListEqual(lowerCAmelCase__    ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__  )    ,					[465, 287, 265, 631, 842]  )

							_lowerCAmelCase						:	Optional[Any]        =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
							# fmt: off
							self.assertListEqual(
							    lowerCAmelCase__    ,					["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]    ,					)
							# fmt: on

							_lowerCAmelCase						:	List[str]        =					tokenizer.convert_tokens_to_ids(lowerCAmelCase__  )
							self.assertListEqual(
							    lowerCAmelCase__    ,					[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]    ,					)

							_lowerCAmelCase						:	Union[str, Any]        =					tokenizer.convert_ids_to_tokens(lowerCAmelCase__  )
							# fmt: off
							self.assertListEqual(
							    lowerCAmelCase__    ,					["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]  )
							# fmt: on






					def    __A  (		self  ):
							_lowerCAmelCase						:	Optional[int]        =					GPTSwaTokenizer(lowerCAmelCase__  )
							_lowerCAmelCase						:	int        =					["This is a test", "I was born in 92000, and this is falsé."]
							_lowerCAmelCase						:	int        =					[
							    [465, 287, 265, 631, 842],
							    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
							]

							# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
							for text, expected_ids in zip(lowerCAmelCase__    ,					lowerCAmelCase__  ):
									self.assertListEqual(tokenizer.encode_fast(lowerCAmelCase__  )    ,					lowerCAmelCase__  )

							# Test that decode_fast returns the input text
							for text, token_ids in zip(lowerCAmelCase__    ,					lowerCAmelCase__  ):
									self.assertEqual(tokenizer.decode_fast(lowerCAmelCase__  )    ,					lowerCAmelCase__  )






					@slow
					def    __A  (		self  ):
							_lowerCAmelCase						:	Optional[Any]        =					[
							    "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
							    "Hey there, how are you doing this fine day?",
							    "This is a text with a trailing spaces followed by a dot     .",
							    "Häj sväjs lillebrör! =)",
							    "Det är inget fel på Mr. Cool",
							]

							# fmt: off
							_lowerCAmelCase						:	Optional[Any]        =					{"input_ids": [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
							# fmt: on
							self.tokenizer_integration_test_util(
							    expected_encoding=lowerCAmelCase__    ,					model_name="""AI-Sweden/gpt-sw3-126m"""    ,					sequences=lowerCAmelCase__    ,					)





"""simple docstring"""




from __future__ import annotations





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	list     )	->				float:
  if not nums:
    raise ValueError("""List is empty"""     )
  return sum(_lowerCamelCase     ) / len(_lowerCamelCase     )


if __name__ == "__main__":
     import doctest

     doctest.testmod()




