


"""simple docstring"""



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  __snake_case             =   """pt"""
elif is_tf_available():
  __snake_case             =   """tf"""
else:
  __snake_case             =   """jax"""







class _lowerCAmelCase    (				snake_case_   ,							unittest.TestCase     ):
   __UpperCAmelCase			:	str							  =  ByTaTokenizer
   __UpperCAmelCase			:	Any							  =  False







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       super().setUp()
       snake_case   :				Dict				 =   ByTaTokenizer()
       tokenizer.save_pretrained(self.tmpdirname						)







   @cached_property
   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       return ByTaTokenizer.from_pretrained("google/byt5-small"						)







   def 			lowerCamelCase		(     self						,     **UpperCamelCase__						)     ->		ByTaTokenizer:

       '''simple docstring'''




       return self.tokenizer_class.from_pretrained(self.tmpdirname						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__=False						,     UpperCamelCase__=20						,     UpperCamelCase__=5						)     ->		Tuple[str, list]:

       '''simple docstring'''




       snake_case   :				Any				 =   []
       for i in range(len(UpperCamelCase__						)						):
           try:
               snake_case   :				List[Any]				 =   tokenizer.decode([i]						,     clean_up_tokenization_spaces=UpperCamelCase__						)
           except UnicodeDecodeError:
               pass
           toks.append((i, tok)						)

       snake_case   :				str				 =   list(filter(lambda UpperCamelCase__						: re.match(r"^[ a-zA-Z]+$"						,     t[1]						)						,     UpperCamelCase__						)						)
       snake_case   :				Union[str, Any]				 =   list(filter(lambda UpperCamelCase__						: [t[0]] == tokenizer.encode(t[1]						,     add_special_tokens=UpperCamelCase__						)						,     UpperCamelCase__						)						)
       if max_length is not None and len(UpperCamelCase__						) > max_length:
           snake_case   :				int				 =   toks[:max_length]
       if min_length is not None and len(UpperCamelCase__						) < min_length and len(UpperCamelCase__						) > 0:
           while len(UpperCamelCase__						) < min_length:
               snake_case   :				int				 =   toks + toks
        # toks_str = [t[1] for t in toks]
       snake_case   :				str				 =   [t[0] for t in toks]

       # Ensure consistency
       snake_case   :				Optional[int]				 =   tokenizer.decode(UpperCamelCase__						,     clean_up_tokenization_spaces=UpperCamelCase__						)
       if " " not in output_txt and len(UpperCamelCase__						) > 1:
           snake_case   :				Union[str, Any]				 =   (
               tokenizer.decode([toks_ids[0]]						,     clean_up_tokenization_spaces=UpperCamelCase__						)
               + " "
               + tokenizer.decode(toks_ids[1:]						,     clean_up_tokenization_spaces=UpperCamelCase__						)
           )
       if with_prefix_space:
           snake_case   :				List[Any]				 =   " " + output_txt
       snake_case   :				Tuple				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
       return output_txt, output_ids







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Any				 =   self.ta_base_tokenizer
       snake_case   :				Tuple				 =   tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]						)
       snake_case   :				Dict				 =   tokenizer(["hi", "I went to the gym", ""]						)
       self.assertListEqual(batch_with_eos_added["input_ids"]						,     batch_without_eos_added["input_ids"]						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   self.ta_base_tokenizer
       snake_case   :				List[Any]				 =   "Unicode €."
       snake_case   :				Any				 =   tokenizer(UpperCamelCase__						)
       snake_case   :				int				 =   [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
       self.assertEqual(encoded["input_ids"]						,     UpperCamelCase__						)

       # decoding
       snake_case   :				List[str]				 =   tokenizer.decode(UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,     "Unicode €.</s>"						)

       snake_case   :				Tuple				 =   tokenizer("e è é ê ë"						)
       snake_case   :				Optional[Any]				 =   [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
       self.assertEqual(encoded["input_ids"]						,     UpperCamelCase__						)
       # decoding
       snake_case   :				Union[str, Any]				 =   tokenizer.decode(UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,     "e è é ê ë</s>"						)

       # encode/decode, but with `encode` instead of `__call__`
       self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"						)						)						,     "e è é ê ë</s>"						)







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				List[str]				 =   self.ta_base_tokenizer
       snake_case   :				int				 =   ["A long paragraph for summarization.", "Another paragraph for summarization."]
       # fmt: off
       snake_case   :				Any				 =   [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
       # fmt: on
       snake_case   :				Union[str, Any]				 =   tokenizer(UpperCamelCase__						,     padding=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)
       self.assertIsInstance(UpperCamelCase__						,     UpperCamelCase__						)

       if FRAMEWORK != "jax":
           snake_case   :				Dict				 =   list(batch.input_ids.numpy()[0]						)
       else:
           snake_case   :				Union[str, Any]				 =   list(batch.input_ids.tolist()[0]						)

       self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

       self.assertEqual((2, 37)						,     batch.input_ids.shape						)
       self.assertEqual((2, 37)						,     batch.attention_mask.shape						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[int]				 =   self.ta_base_tokenizer
       snake_case   :				Union[str, Any]				 =   ["A long paragraph for summarization.", "Another paragraph for summarization."]
       snake_case   :				int				 =   tokenizer(UpperCamelCase__						,     padding=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)
       # check if input_ids are returned and no decoder_input_ids
       self.assertIn("input_ids"						,     UpperCamelCase__						)
       self.assertIn("attention_mask"						,     UpperCamelCase__						)
       self.assertNotIn("decoder_input_ids"						,     UpperCamelCase__						)
       self.assertNotIn("decoder_attention_mask"						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Tuple:

       '''simple docstring'''




       snake_case   :				int				 =   self.ta_base_tokenizer
       snake_case   :				Union[str, Any]				 =   [
           "Summary of the text.",
           "Another summary.",
       ]
       snake_case   :				Optional[int]				 =   tokenizer(
           text_target=UpperCamelCase__						,     max_length=32						,     padding="max_length"						,     truncation=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)
       self.assertEqual(32						,     targets["input_ids"].shape[1]						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Tuple				 =   self.ta_base_tokenizer
       snake_case   :				Tuple				 =   ["A long paragraph for summarization. </s>"]
       snake_case   :				Tuple				 =   ["Summary of the text. </s>"]
       # fmt: off
       snake_case   :				Optional[Any]				 =   [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
       snake_case   :				Any				 =   [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
       # fmt: on

       snake_case   :				str				 =   tokenizer(UpperCamelCase__						,     text_target=UpperCamelCase__						)

       self.assertEqual(UpperCamelCase__						,     batch["input_ids"][0]						)
       self.assertEqual(UpperCamelCase__						,     batch["labels"][0]						)







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   self.get_tokenizers()
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               self.assertNotEqual(tokenizer.model_max_length						,     42						)

        # Now let's start the test
       snake_case   :				List[Any]				 =   self.get_tokenizers()
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               # Isolate this from the other tests because we save additional tokens/etc
               snake_case   :				Union[str, Any]				 =   tempfile.mkdtemp()

               snake_case   :				Union[str, Any]				 =   " He is very happy, UNwant\u00E9d,running"
               snake_case   :				Any				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
               tokenizer.save_pretrained(UpperCamelCase__						)

               snake_case   :				Any				 =   tokenizer.__class__.from_pretrained(UpperCamelCase__						)
               snake_case   :				str				 =   after_tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
               self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)

               shutil.rmtree(UpperCamelCase__						)

       snake_case   :				Optional[Any]				 =   self.get_tokenizers(model_max_length=42						)
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               # Isolate this from the other tests because we save additional tokens/etc
               snake_case   :				Union[str, Any]				 =   tempfile.mkdtemp()

               snake_case   :				List[Any]				 =   " He is very happy, UNwant\u00E9d,running"
               tokenizer.add_tokens(["bim", "bambam"]						)
               snake_case   :				List[str]				 =   tokenizer.additional_special_tokens
               additional_special_tokens.append("new_additional_special_token"						)
               tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}						)
               snake_case   :				Tuple				 =   tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
               tokenizer.save_pretrained(UpperCamelCase__						)

               snake_case   :				Any				 =   tokenizer.__class__.from_pretrained(UpperCamelCase__						)
               snake_case   :				Union[str, Any]				 =   after_tokenizer.encode(UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						)
               self.assertListEqual(UpperCamelCase__						,     UpperCamelCase__						)
               self.assertIn("new_additional_special_token"						,     after_tokenizer.additional_special_tokens						)
               self.assertEqual(after_tokenizer.model_max_length						,     42						)

               snake_case   :				Optional[Any]				 =   tokenizer.__class__.from_pretrained(UpperCamelCase__						,     model_max_length=43						)
               self.assertEqual(tokenizer.model_max_length						,     43						)

               shutil.rmtree(UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   []
       if self.test_slow_tokenizer:
           tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

       if self.test_rust_tokenizer:
           tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

       for tokenizer_class, tokenizer_utils in tokenizer_list:
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer_utils.save_pretrained(UpperCamelCase__						)

               with open(os.path.join(UpperCamelCase__						,     "special_tokens_map.json"						)						,     encoding="utf-8"						) as json_file:
                   snake_case   :				str				 =   json.load(UpperCamelCase__						)

               with open(os.path.join(UpperCamelCase__						,     "tokenizer_config.json"						)						,     encoding="utf-8"						) as json_file:
                   snake_case   :				Any				 =   json.load(UpperCamelCase__						)

               snake_case   :				Union[str, Any]				 =   [F'<extra_id_{i}>' for i in range(125						)]

               snake_case   :				Optional[int]				 =   added_tokens_extra_ids + [
                   "an_additional_special_token"
               ]
               snake_case   :				int				 =   added_tokens_extra_ids + [
                   "an_additional_special_token"
               ]

               with open(os.path.join(UpperCamelCase__						,     "special_tokens_map.json"						)						,     "w"						,     encoding="utf-8"						) as outfile:
                   json.dump(UpperCamelCase__						,     UpperCamelCase__						)
               with open(os.path.join(UpperCamelCase__						,     "tokenizer_config.json"						)						,     "w"						,     encoding="utf-8"						) as outfile:
                   json.dump(UpperCamelCase__						,     UpperCamelCase__						)

               # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
               # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
               # "special_tokens_map.json" files
               snake_case   :				List[Any]				 =   tokenizer_class.from_pretrained(
                   UpperCamelCase__						,     )
               self.assertIn(
                   "an_additional_special_token"						,     tokenizer_without_change_in_init.additional_special_tokens						)
               # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
               self.assertEqual(
                   ["an_additional_special_token"]						,     tokenizer_without_change_in_init.convert_ids_to_tokens(
                       tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]						)						)						,     )

               # Now we test that we can change the value of additional_special_tokens in the from_pretrained
               snake_case   :				Optional[int]				 =   added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"						,     lstrip=UpperCamelCase__						)]
               snake_case   :				Any				 =   tokenizer_class.from_pretrained(
                   UpperCamelCase__						,     additional_special_tokens=UpperCamelCase__						,     )

               self.assertIn("a_new_additional_special_token"						,     tokenizer.additional_special_tokens						)
               self.assertEqual(
                   ["a_new_additional_special_token"]						,     tokenizer.convert_ids_to_tokens(
                       tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]						)						)						,     )







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   []
       if self.test_slow_tokenizer:
           tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

       if self.test_rust_tokenizer:
           tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

       for tokenizer_class, tokenizer_utils in tokenizer_list:
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer_utils.save_pretrained(UpperCamelCase__						)

               snake_case   :				str				 =   tokenizer_class.from_pretrained(UpperCamelCase__						)

               self.assertTrue(tokenizer.decode([255]						) == ""						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       pass







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				int				 =   self.get_tokenizers(fast=UpperCamelCase__						,     do_lower_case=UpperCamelCase__						)
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               snake_case   :				List[str]				 =   ["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
               snake_case   :				Union[str, Any]				 =   tokenizer.convert_tokens_to_string(UpperCamelCase__						)

               self.assertIsInstance(UpperCamelCase__						,     UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   self.get_tokenizers()
       for tokenizer in tokenizers:
           with self.subTest(F'{tokenizer.__class__.__name__}'						):
               snake_case   :				Optional[int]				 =   [
                   "bos_token",
                   "eos_token",
                   "unk_token",
                   "sep_token",
                   "pad_token",
                   "cls_token",
                   "mask_token",
               ]

               snake_case   :				Optional[Any]				 =   0
               snake_case   :				Optional[Any]				 =   tokenizer.convert_ids_to_tokens(
                   UpperCamelCase__						,     skip_special_tokens=UpperCamelCase__						)

               for attr in attributes_list:
                   setattr(UpperCamelCase__						,     attr + "_id"						,     UpperCamelCase__						)
                   self.assertEqual(getattr(UpperCamelCase__						,     UpperCamelCase__						)						,     UpperCamelCase__						)
                   self.assertEqual(getattr(UpperCamelCase__						,     attr + "_id"						)						,     UpperCamelCase__						)

                   setattr(UpperCamelCase__						,     attr + "_id"						,     UpperCamelCase__						)
                   self.assertEqual(getattr(UpperCamelCase__						,     UpperCamelCase__						)						,     UpperCamelCase__						)
                   self.assertEqual(getattr(UpperCamelCase__						,     attr + "_id"						)						,     UpperCamelCase__						)

               setattr(UpperCamelCase__						,     "additional_special_tokens_ids"						,     []						)
               self.assertListEqual(getattr(UpperCamelCase__						,     "additional_special_tokens"						)						,     []						)
               self.assertListEqual(getattr(UpperCamelCase__						,     "additional_special_tokens_ids"						)						,     []						)

               setattr(UpperCamelCase__						,     "additional_special_tokens_ids"						,     [token_id_to_test_setters]						)
               self.assertListEqual(getattr(UpperCamelCase__						,     "additional_special_tokens"						)						,     [token_to_test_setters]						)
               self.assertListEqual(getattr(UpperCamelCase__						,     "additional_special_tokens_ids"						)						,     [token_id_to_test_setters]						)






"""simple docstring"""



import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
  import tensorflow as tf
  from tensorflow.python.eager import context
  from tensorflow.python.framework import ops

  from transformers import GradientAccumulator, create_optimizer







@require_tf
class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Optional[Any]:

       '''simple docstring'''




       self.assertEqual(len(UpperCamelCase__						)						,     len(UpperCamelCase__						)						)
       for a, b in zip(UpperCamelCase__						,     UpperCamelCase__						):
           self.assertAlmostEqual(UpperCamelCase__						,     UpperCamelCase__						,     delta=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   GradientAccumulator()
       accumulator([tf.constant([1.0, 2.0]						)]						)
       accumulator([tf.constant([-2.0, 1.0]						)]						)
       accumulator([tf.constant([-1.0, 2.0]						)]						)
       with self.assertRaises(UpperCamelCase__						):
           accumulator([tf.constant([1.0, 1.0]						), tf.constant([2.0, 2.0]						)]						)
       self.assertEqual(accumulator.step						,     3						)
       self.assertEqual(len(accumulator.gradients						)						,     1						)
       self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()						,     [-2.0, 5.0]						,     tol=1e-2						)
       accumulator.reset()
       self.assertEqual(accumulator.step						,     0						)
       self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()						,     [0.0, 0.0]						,     tol=1e-2						)







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   None
       ops.enable_eager_execution_internal()
       snake_case   :				str				 =   tf.config.list_physical_devices("CPU"						)
       if len(UpperCamelCase__						) == 1:
           tf.config.set_logical_device_configuration(
               physical_devices[0]						,     [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]						)
       snake_case   :				Optional[int]				 =   tf.config.list_logical_devices(device_type="CPU"						)
       snake_case   :				List[str]				 =   tf.distribute.MirroredStrategy(devices=devices[:2]						)

       with strategy.scope():
           snake_case   :				int				 =   GradientAccumulator()
           snake_case   :				int				 =   tf.Variable([4.0, 3.0]						)
           snake_case ,snake_case   :				Any				 =   create_optimizer(5e-5						,     10						,     5						)
           snake_case   :				Tuple				 =   tf.Variable([0.0, 0.0]						,     trainable=UpperCamelCase__						)

       def accumulate_on_replica(UpperCamelCase__						):
           accumulator([gradient]						)

       def apply_on_replica():
           optimizer.apply_gradients(list(zip(accumulator.gradients						,     [variable]						)						)						)

       @tf.function
       def accumulate(UpperCamelCase__						,     UpperCamelCase__						):
           with strategy.scope():
               snake_case   :				Union[str, Any]				 =   strategy.experimental_local_results(UpperCamelCase__						)
               local_variables[0].assign(UpperCamelCase__						)
               local_variables[1].assign(UpperCamelCase__						)
               strategy.run(UpperCamelCase__						,     args=(gradient_placeholder,)						)

       @tf.function
       def apply_grad():
           with strategy.scope():
               strategy.run(UpperCamelCase__						)

       def _check_local_values(UpperCamelCase__						,     UpperCamelCase__						):
           snake_case   :				List[Any]				 =   strategy.experimental_local_results(accumulator._gradients[0]						)
           self.assertListAlmostEqual(values[0].value()						,     UpperCamelCase__						,     tol=1e-2						)
           self.assertListAlmostEqual(values[1].value()						,     UpperCamelCase__						,     tol=1e-2						)

       accumulate([1.0, 2.0]						,     [-1.0, 1.0]						)
       accumulate([3.0, -1.0]						,     [-1.0, -1.0]						)
       accumulate([-2.0, 2.0]						,     [3.0, -2.0]						)
       self.assertEqual(accumulator.step						,     3						)
       _check_local_values([2.0, 3.0]						,     [1.0, -2.0]						)
       apply_grad()
       self.assertListAlmostEqual(variable.value()						,     [4.0, 3.0]						,     tol=1e-2						)
       accumulator.reset()
       self.assertEqual(accumulator.step						,     0						)
       _check_local_values([0.0, 0.0]						,     [0.0, 0.0]						)





