"""simple docstring"""




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 						__A							(			SCREAMING_SNAKE_CASE_   ,     unittest.TestCase		):
     _UpperCamelCase							:      Tuple		 =		GPTSanJapaneseTokenizer
     _UpperCamelCase							:      Tuple		 =		False
     _UpperCamelCase							:      Tuple		 =		{"do_clean_text": False, "add_prefix_space": False}






     def    __A  (		self  ):
       super().setUp()

       # fmt: off
       _lowerCAmelCase						:	List[Any]        =					["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
       # fmt: on
       _lowerCAmelCase						:	Optional[Any]        =					{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
       _lowerCAmelCase						:	Optional[int]        =					{"""unk_token""": """<unk>"""}

       _lowerCAmelCase						:	int        =					os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["""vocab_file"""]  )
       _lowerCAmelCase						:	Tuple        =					os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["""emoji_file"""]  )
       with open(self.vocab_file    ,					"""w"""    ,					encoding="""utf-8"""  ) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )
       with open(self.emoji_file    ,					"""w"""  ) as emoji_writer:
         emoji_writer.write(json.dumps(a__  )  )






     def    __A  (		self    ,					**a__  ):
       kwargs.update(self.special_tokens_map  )
       return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname    ,					**a__  )






     def    __A  (		self    ,					a__  ):
       _lowerCAmelCase						:	str        =					"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
       _lowerCAmelCase						:	Optional[int]        =					"""こんにちは、世界。 \nこんばんは、世界。😀"""
       return input_text, output_text






     def    __A  (		self    ,					a__  ):
       _lowerCAmelCase      ,     _lowerCAmelCase						:	Dict        =					self.get_input_output_texts(a__  )
       _lowerCAmelCase						:	Dict        =					tokenizer.encode(a__    ,					add_special_tokens=a__  )
       _lowerCAmelCase						:	List[str]        =					tokenizer.decode(a__    ,					clean_up_tokenization_spaces=a__  )
       return text, ids






     def    __A  (		self  ):
       pass  # TODO add if relevant






     def    __A  (		self  ):
       pass  # TODO add if relevant






     def    __A  (		self  ):
       pass  # TODO add if relevant






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					self.get_tokenizer()

       # Testing tokenization
       _lowerCAmelCase						:	Dict        =					"""こんにちは、世界。　こんばんは、㔺界。"""
       _lowerCAmelCase						:	str        =					["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
       _lowerCAmelCase						:	str        =					tokenizer.tokenize(a__  )
       self.assertListEqual(a__    ,					a__  )

       # Testing conversion to ids without special tokens
       _lowerCAmelCase						:	List[str]        =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.convert_tokens_to_ids(a__  )
       self.assertListEqual(a__    ,					a__  )

       # Testing conversion to ids with special tokens
       _lowerCAmelCase						:	Any        =					tokens + [tokenizer.unk_token]
       _lowerCAmelCase						:	Dict        =					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.convert_tokens_to_ids(a__  )
       self.assertListEqual(a__    ,					a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					self.get_tokenizer()

       # Testing tokenization
       _lowerCAmelCase						:	Optional[Any]        =					"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
       _lowerCAmelCase						:	Optional[int]        =					"""こんにちは、、、、世界。こんばんは、、、、世界。"""
       _lowerCAmelCase						:	Any        =					tokenizer.encode(a__  )
       _lowerCAmelCase						:	str        =					tokenizer.decode(a__  )
       self.assertEqual(a__    ,					a__  )






     @slow
     def    __A  (		self  ):
       _lowerCAmelCase						:	int        =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""  )

       # Testing tokenization
       _lowerCAmelCase						:	Optional[Any]        =					"""こんにちは、世界。"""
       _lowerCAmelCase						:	Optional[int]        =					"""こんばんは、㔺界。😀"""
       _lowerCAmelCase						:	int        =					"""こんにちは、世界。こんばんは、世界。😀"""
       _lowerCAmelCase						:	List[Any]        =					tokenizer.encode(prefix_text + input_text  )
       _lowerCAmelCase						:	Tuple        =					tokenizer.encode(""""""    ,					prefix_text=prefix_text + input_text  )
       _lowerCAmelCase						:	Tuple        =					tokenizer.encode(a__    ,					prefix_text=a__  )
       _lowerCAmelCase						:	int        =					tokenizer.decode(a__  )
       _lowerCAmelCase						:	Tuple        =					tokenizer.decode(a__  )
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.decode(a__  )
       self.assertEqual(a__    ,					a__  )
       self.assertEqual(a__    ,					a__  )
       self.assertEqual(a__    ,					a__  )






     @slow
     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""  )

       # Testing tokenization
       _lowerCAmelCase						:	Union[str, Any]        =					"""こんにちは、世界。"""
       _lowerCAmelCase						:	Union[str, Any]        =					"""こんばんは、㔺界。😀"""

       _lowerCAmelCase						:	Optional[Any]        =					len(tokenizer.encode(a__  )  ) - 2
       _lowerCAmelCase						:	int        =					len(tokenizer.encode(a__  )  ) - 2

       _lowerCAmelCase						:	Union[str, Any]        =					[1] + [0] * (len_prefix + len_text + 1)
       _lowerCAmelCase						:	Tuple        =					[1] * (len_prefix + len_text + 1) + [0]
       _lowerCAmelCase						:	Optional[int]        =					[1] + [1] * (len_prefix) + [0] * (len_text + 1)

       _lowerCAmelCase						:	List[str]        =					tokenizer(prefix_text + input_text  ).token_type_ids
       _lowerCAmelCase						:	List[Any]        =					tokenizer(""""""    ,					prefix_text=prefix_text + input_text  ).token_type_ids
       _lowerCAmelCase						:	Tuple        =					tokenizer(a__    ,					prefix_text=a__  ).token_type_ids
       self.assertListEqual(a__    ,					a__  )
       self.assertListEqual(a__    ,					a__  )
       self.assertListEqual(a__    ,					a__  )






     @slow
     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[int]        =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""  )

       _lowerCAmelCase						:	Optional[int]        =					tokenizer.encode("""あンいワ"""  )
       _lowerCAmelCase						:	Tuple        =					tokenizer.encode(""""""    ,					prefix_text="""あンいワ"""  )
       _lowerCAmelCase						:	List[str]        =					tokenizer.encode("""いワ"""    ,					prefix_text="""あン"""  )

       self.assertEqual(tokenizer.decode(a__  )    ,					tokenizer.decode(a__  )  )
       self.assertEqual(tokenizer.decode(a__  )    ,					tokenizer.decode(a__  )  )
       self.assertNotEqual(a__    ,					a__  )
       self.assertNotEqual(a__    ,					a__  )
       self.assertEqual(x_token_a[1]    ,					x_token_a[-1]  )  # SEG token
       self.assertEqual(x_token_a[1]    ,					x_token_a[3]  )  # SEG token






     @slow
     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""  )

       _lowerCAmelCase						:	Any        =					[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
       _lowerCAmelCase						:	str        =					tokenizer(a__    ,					padding=a__  )
       _lowerCAmelCase						:	Tuple        =					tokenizer.batch_encode_plus(a__    ,					padding=a__  )

       # fmt: off
       _lowerCAmelCase						:	Dict        =					[[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
       _lowerCAmelCase						:	Optional[Any]        =					[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
       _lowerCAmelCase						:	str        =					[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
       # fmt: on
       self.assertListEqual(x_token.input_ids    ,					a__  )
       self.assertListEqual(x_token.token_type_ids    ,					a__  )
       self.assertListEqual(x_token.attention_mask    ,					a__  )
       self.assertListEqual(x_token_a.input_ids    ,					a__  )
       self.assertListEqual(x_token_a.token_type_ids    ,					a__  )
       self.assertListEqual(x_token_a.attention_mask    ,					a__  )






     def    __A  (		self  ):
       # Intentionally convert some words to accommodate character fluctuations unique to Japanese
       pass






     def    __A  (		self  ):
       # tokenizer has no padding token
       pass





"""simple docstring"""




_a				:		Any	   =       '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

_a				:		List[Any]	   =       [{'type': 'code', 'content': INSTALL_CONTENT}]
_a				:		Union[str, Any]	   =       {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}




