

"""simple docstring"""





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast




class  _lowerCAmelCase  (     unittest.TestCase       ):







 """simple docstring"""







 def  _lowercase			(      self							:      Optional[Any]     ):
    __lowercase								=						mock.Mock()
    __lowercase								=						5_0_0
    __lowercase								=						{}
    __lowercase								=						HTTPError
    __lowercase								=						{}

    # Download this model to make sure it's in the cache.
    __lowercase								=						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )

    # Under the mock environment we get a 500 error when trying to reach the tokenizer.
    with mock.patch("requests.Session.request", return_value=__a     ) as mock_head:
       __lowercase								=						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"     )
       # This check we did call the fake head request
       mock_head.assert_called()


 @require_tokenizers
 def  _lowercase			(      self							:      List[Any]     ):
    __lowercase								=						mock.Mock()
    __lowercase								=						5_0_0
    __lowercase								=						{}
    __lowercase								=						HTTPError
    __lowercase								=						{}

    # Download this model to make sure it's in the cache.
    __lowercase								=						GPTaTokenizerFast.from_pretrained("gpt2"     )

    # Under the mock environment we get a 500 error when trying to reach the tokenizer.
    with mock.patch("requests.Session.request", return_value=__a     ) as mock_head:
       __lowercase								=						GPTaTokenizerFast.from_pretrained("gpt2"     )
       # This check we did call the fake head request
       mock_head.assert_called()


 def  _lowercase			(      self							:      List[Any]     ):
    # This test is for deprecated behavior and can be removed in v5
    try:
       __lowercase								=						tempfile.mktemp()
       with open(__a, "wb"     ) as f:
          http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model", __a     )

       __lowercase								=						AlbertTokenizer.from_pretrained(__a     )
    finally:
       os.remove(__a     )

    # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
    # the current folder and have the right name.
    if os.path.isfile("tokenizer.json"     ):
       # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
       return
    try:
       with open("tokenizer.json", "wb"     ) as f:
          http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json", __a     )
       __lowercase								=						AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"     )
       # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
       self.assertEqual(tokenizer.vocab_size, 1_0_0_0     )
       # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

    finally:
       os.remove("tokenizer.json"     )



 def  _lowercase			(      self							:      List[str]     ):
    __lowercase								=						AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"     )







@is_staging_test
class  _lowerCAmelCase  (     unittest.TestCase       ):







 """simple docstring"""







 __UpperCAmelCase     : Tuple       		=     ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]


 @classmethod
 def  _lowercase			(      cls							:      int     ):
    __lowercase								=						TOKEN
    HfFolder.save_token(__a     )


 @classmethod
 def  _lowercase			(      cls							:      List[Any]     ):
    try:
       delete_repo(token=cls._token, repo_id="test-tokenizer"     )
    except HTTPError:
       pass

    try:
       delete_repo(token=cls._token, repo_id="valid_org/test-tokenizer-org"     )
    except HTTPError:
       pass

    try:
       delete_repo(token=cls._token, repo_id="test-dynamic-tokenizer"     )
    except HTTPError:
       pass


 def  _lowercase			(      self							:      List[Any]     ):
    with tempfile.TemporaryDirectory() as tmp_dir:
       __lowercase								=						os.path.join(__a, "vocab.txt"     )
       with open(__a, "w", encoding="utf-8"     ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
       __lowercase								=						BertTokenizer(__a     )

    tokenizer.push_to_hub("test-tokenizer", use_auth_token=self._token     )
    __lowercase								=						BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""     )
    self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab     )

    # Reset repo
    delete_repo(token=self._token, repo_id="test-tokenizer"     )

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(__a, repo_id="test-tokenizer", push_to_hub=__a, use_auth_token=self._token     )

    __lowercase								=						BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""     )
    self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab     )


 def  _lowercase			(      self							:      Optional[int]     ):
    with tempfile.TemporaryDirectory() as tmp_dir:
       __lowercase								=						os.path.join(__a, "vocab.txt"     )
       with open(__a, "w", encoding="utf-8"     ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
       __lowercase								=						BertTokenizer(__a     )

    tokenizer.push_to_hub("valid_org/test-tokenizer-org", use_auth_token=self._token     )
    __lowercase								=						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"     )
    self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab     )

    # Reset repo
    delete_repo(token=self._token, repo_id="valid_org/test-tokenizer-org"     )

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(
           __a, repo_id="valid_org/test-tokenizer-org", push_to_hub=__a, use_auth_token=self._token     )

    __lowercase								=						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"     )
    self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab     )



 @require_tokenizers
 def  _lowercase			(      self							:      Optional[Any]     ):
    CustomTokenizer.register_for_auto_class()
    with tempfile.TemporaryDirectory() as tmp_dir:
       __lowercase								=						os.path.join(__a, "vocab.txt"     )
       with open(__a, "w", encoding="utf-8"     ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )
       __lowercase								=						CustomTokenizer(__a     )

    # No fast custom tokenizer
    tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token     )

    __lowercase								=						AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer""", trust_remote_code=__a     )
    # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer"     )

    # Fast and slow custom tokenizer
    CustomTokenizerFast.register_for_auto_class()
    with tempfile.TemporaryDirectory() as tmp_dir:
       __lowercase								=						os.path.join(__a, "vocab.txt"     )
       with open(__a, "w", encoding="utf-8"     ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]     )     )

       __lowercase								=						BertTokenizerFast.from_pretrained(__a     )
       bert_tokenizer.save_pretrained(__a     )
       __lowercase								=						CustomTokenizerFast.from_pretrained(__a     )

    tokenizer.push_to_hub("test-dynamic-tokenizer", use_auth_token=self._token     )

    __lowercase								=						AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer""", trust_remote_code=__a     )
    # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizerFast"     )
    __lowercase								=						AutoTokenizer.from_pretrained(
        F"""{USER}/test-dynamic-tokenizer""", use_fast=__a, trust_remote_code=__a     )
    # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
    self.assertEqual(tokenizer.__class__.__name__, "CustomTokenizer"     )







class  _lowerCAmelCase  (     unittest.TestCase       ):







 """simple docstring"""







 def  _lowercase			(      self							:      str     ):
    __lowercase								=						Trie()
    trie.add("Hello 友達"     )
    self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}     )
    trie.add("Hello"     )
    trie.data
    self.assertEqual(trie.data, {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}     )


 def  _lowercase			(      self							:      Optional[Any]     ):
    __lowercase								=						Trie()
    self.assertEqual(trie.split("[CLS] This is a extra_id_100"     ), ["[CLS] This is a extra_id_100"]     )
    trie.add("[CLS]"     )
    trie.add("extra_id_1"     )
    trie.add("extra_id_100"     )
    self.assertEqual(trie.split("[CLS] This is a extra_id_100"     ), ["[CLS]", " This is a ", "extra_id_100"]     )


 def  _lowercase			(      self							:      int     ):
    __lowercase								=						Trie()
    trie.add("A"     )
    self.assertEqual(trie.split("ABC"     ), ["A", "BC"]     )
    self.assertEqual(trie.split("BCA"     ), ["BC", "A"]     )


 def  _lowercase			(      self							:      List[Any]     ):
    __lowercase								=						Trie()
    trie.add("TOKEN]"     )
    trie.add("[SPECIAL_TOKEN]"     )
    self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"     ), ["This is something ", "[SPECIAL_TOKEN]"]     )


 def  _lowercase			(      self							:      str     ):
    __lowercase								=						Trie()
    trie.add("A"     )
    trie.add("P"     )
    trie.add("[SPECIAL_TOKEN]"     )
    self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"     ), ["This is something ", "[SPECIAL_TOKEN]"]     )


 def  _lowercase			(      self							:      Dict     ):
    __lowercase								=						Trie()
    trie.add("AB"     )
    trie.add("B"     )
    trie.add("C"     )
    self.assertEqual(trie.split("ABC"     ), ["AB", "C"]     )


 def  _lowercase			(      self							:      Optional[Any]     ):
    __lowercase								=						Trie()
    trie.add("ABC"     )
    trie.add("B"     )
    trie.add("CD"     )
    self.assertEqual(trie.split("ABCD"     ), ["ABC", "D"]     )



 def  _lowercase			(      self							:      Union[str, Any]     ):
    __lowercase								=						Trie()
    __lowercase								=						trie.cut_text("ABC", [0, 0, 2, 1, 2, 3]     )
    self.assertEqual(__a, ["AB", "C"]     )







"""simple docstring"""





import argparse
import os
import re


_a							=							'src/transformers/models/auto'


# re pattern that matches mapping introductions:
#    SUPER_MODEL_MAPPING_NAMES = OrderedDict or SUPER_MODEL_MAPPING = OrderedDict
_a							=							re.compile(r'[A-Z_]+_MAPPING(\s+|_[A-Z_]+\s+)=\s+OrderedDict')
# re pattern that matches identifiers in mappings
_a							=							re.compile(r'\s*\(\s*"(\S[^"]+)"')




def   _A  (  UpperCamelCase_					:	Any,      UpperCamelCase_					:	bool = False)     ->						Optional[Any]:
   '''simple docstring'''


   with open(UpperCamelCase_,      "r",      encoding="utf-8") as f:
      __lowercase								=						f.read()

   __lowercase								=						content.split("\n")
   __lowercase								=						[]
   __lowercase								=						0
   while line_idx < len(UpperCamelCase_):
      if _re_intro_mapping.search(lines[line_idx]) is not None:
         __lowercase								=						len(re.search(r"^(\s*)\S",      lines[line_idx]).groups()[0]) + 8
         # Start of a new mapping!
         while not lines[line_idx].startswith(" " * indent + "("):
            new_lines.append(lines[line_idx])
            line_idx += 1

         __lowercase								=						[]
         while lines[line_idx].strip() != "]":
            # Blocks either fit in one line or not
            if lines[line_idx].strip() == "(":
               __lowercase								=						line_idx
               while not lines[line_idx].startswith(" " * indent + ")"):
                  line_idx += 1
               blocks.append("\n".join(lines[start_idx : line_idx + 1]))
            else:
               blocks.append(lines[line_idx])
            line_idx += 1

         # Sort blocks by their identifiers
         __lowercase								=						sorted(UpperCamelCase_,      key=lambda UpperCamelCase_: _re_identifier.search(UpperCamelCase_).groups()[0])
         new_lines += blocks
      else:
         new_lines.append(lines[line_idx])
         line_idx += 1

   if overwrite:
      with open(UpperCamelCase_,      "w",      encoding="utf-8") as f:
         f.write("\n".join(UpperCamelCase_))
   elif "\n".join(UpperCamelCase_) != content:
      return True




def   _A  (  UpperCamelCase_					:	bool = False)     ->						Optional[Any]:
   '''simple docstring'''


   __lowercase								=						[os.path.join(UpperCamelCase_,      UpperCamelCase_) for f in os.listdir(UpperCamelCase_) if f.endswith(".py")]
   __lowercase								=						[sort_auto_mapping(UpperCamelCase_,      overwrite=UpperCamelCase_) for fname in fnames]

   if not overwrite and any(UpperCamelCase_):
      __lowercase								=						[f for f, d in zip(UpperCamelCase_,      UpperCamelCase_) if d]
      raise ValueError(
          F"""The following files have auto mappings that need sorting: {", ".join(UpperCamelCase_)}. Run `make style` to fix"""
          " this.")


if __name__ == "__main__":
   _a							=							argparse.ArgumentParser()
   parser.add_argument('--check_only', action='store_true', help='Whether to only check or fix style.')
   _a							=							parser.parse_args()

   sort_all_auto_mappings(not args.check_only)




