


'''simple docstring'''






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast






class        __snake_case	(       unittest.TestCase):







    """simple docstring"""



    def __lowercase       (       self   :  str		)       ->  Any:
      lowerCAmelCase_      :				Optional[int]      =       mock.Mock()
      lowerCAmelCase_      :				Any      =       5_00
      lowerCAmelCase_      :				str      =       {}
      lowerCAmelCase_      :				int      =       HTTPError
      lowerCAmelCase_      :				Any      =       {}

      # Download this model to make sure it's in the cache.
      lowerCAmelCase_      :				Tuple      =       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""		)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request"""   ,						return_value=UpperCamelCase__		) as mock_head:
        lowerCAmelCase_      :				Any      =       BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""		)
        # This check we did call the fake head request
        mock_head.assert_called()

    @require_tokenizers
    def __lowercase       (       self   :  Optional[int]		)       ->  Union[str, Any]:
      lowerCAmelCase_      :				str      =       mock.Mock()
      lowerCAmelCase_      :				Tuple      =       5_00
      lowerCAmelCase_      :				Dict      =       {}
      lowerCAmelCase_      :				List[Any]      =       HTTPError
      lowerCAmelCase_      :				Optional[int]      =       {}

      # Download this model to make sure it's in the cache.
      lowerCAmelCase_      :				Optional[Any]      =       GPTaTokenizerFast.from_pretrained("""gpt2"""		)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request"""   ,						return_value=UpperCamelCase__		) as mock_head:
        lowerCAmelCase_      :				str      =       GPTaTokenizerFast.from_pretrained("""gpt2"""		)
        # This check we did call the fake head request
        mock_head.assert_called()

    def __lowercase       (       self   :  Any		)       ->  Union[str, Any]:







      try:
        lowerCAmelCase_      :				Any      =       tempfile.mktemp()
        with open(UpperCamelCase__   ,						"""wb"""		) as f:
          http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""   ,						UpperCamelCase__		)

        lowerCAmelCase_      :				List[str]      =       AlbertTokenizer.from_pretrained(UpperCamelCase__		)
      finally:
        os.remove(UpperCamelCase__		)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile("""tokenizer.json"""		):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
      try:
        with open("""tokenizer.json"""   ,						"""wb"""		) as f:
          http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""   ,						UpperCamelCase__		)
        lowerCAmelCase_      :				Any      =       AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""		)
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size   ,						10_00		)
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
        os.remove("""tokenizer.json"""		)





    def __lowercase       (       self   :  Tuple		)       ->  List[Any]:
      lowerCAmelCase_      :				List[str]      =       AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""		)







@is_staging_test
class        __snake_case	(       unittest.TestCase):







    """simple docstring"""



    lowercase       			= ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']

    @classmethod
    def __lowercase       (       cls   :  int		)       ->  Optional[int]:
      lowerCAmelCase_      :				List[Any]      =       TOKEN
      HfFolder.save_token(UpperCamelCase__		)

    @classmethod
    def __lowercase       (       cls   :  str		)       ->  str:







      try:
        delete_repo(token=cls._token   ,						repo_id="""test-tokenizer"""		)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token   ,						repo_id="""valid_org/test-tokenizer-org"""		)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token   ,						repo_id="""test-dynamic-tokenizer"""		)
      except HTTPError:
        pass

    def __lowercase       (       self   :  str		)       ->  List[Any]:







      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_      :				Any      =       os.path.join(UpperCamelCase__   ,						"""vocab.txt"""		)
        with open(UpperCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""		) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)
        lowerCAmelCase_      :				Dict      =       BertTokenizer(UpperCamelCase__		)

      tokenizer.push_to_hub("""test-tokenizer"""   ,						use_auth_token=self._token		)
      lowerCAmelCase_      :				int      =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
      self.assertDictEqual(new_tokenizer.vocab   ,						tokenizer.vocab		)

      # Reset repo
      delete_repo(token=self._token   ,						repo_id="""test-tokenizer"""		)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(UpperCamelCase__   ,						repo_id="""test-tokenizer"""   ,						push_to_hub=UpperCamelCase__   ,						use_auth_token=self._token		)

      lowerCAmelCase_      :				Optional[Any]      =       BertTokenizer.from_pretrained(F'{USER}/test-tokenizer'		)
      self.assertDictEqual(new_tokenizer.vocab   ,						tokenizer.vocab		)

    def __lowercase       (       self   :  Optional[Any]		)       ->  Union[str, Any]:







      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_      :				Optional[int]      =       os.path.join(UpperCamelCase__   ,						"""vocab.txt"""		)
        with open(UpperCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""		) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)
        lowerCAmelCase_      :				Tuple      =       BertTokenizer(UpperCamelCase__		)

      tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""   ,						use_auth_token=self._token		)
      lowerCAmelCase_      :				str      =       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""		)
      self.assertDictEqual(new_tokenizer.vocab   ,						tokenizer.vocab		)

      # Reset repo
      delete_repo(token=self._token   ,						repo_id="""valid_org/test-tokenizer-org"""		)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            UpperCamelCase__   ,						repo_id="""valid_org/test-tokenizer-org"""   ,						push_to_hub=UpperCamelCase__   ,						use_auth_token=self._token		)

      lowerCAmelCase_      :				Optional[int]      =       BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""		)
      self.assertDictEqual(new_tokenizer.vocab   ,						tokenizer.vocab		)





    @require_tokenizers
    def __lowercase       (       self   :  str		)       ->  Optional[int]:
      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_      :				Dict      =       os.path.join(UpperCamelCase__   ,						"""vocab.txt"""		)
        with open(UpperCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""		) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)
        lowerCAmelCase_      :				int      =       CustomTokenizer(UpperCamelCase__		)

      # No fast custom tokenizer
      tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,						use_auth_token=self._token		)

      lowerCAmelCase_      :				Any      =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'   ,						trust_remote_code=UpperCamelCase__		)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,						"""CustomTokenizer"""		)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_      :				int      =       os.path.join(UpperCamelCase__   ,						"""vocab.txt"""		)
        with open(UpperCamelCase__   ,						"""w"""   ,						encoding="""utf-8"""		) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]		)		)

        lowerCAmelCase_      :				Optional[Any]      =       BertTokenizerFast.from_pretrained(UpperCamelCase__		)
        bert_tokenizer.save_pretrained(UpperCamelCase__		)
        lowerCAmelCase_      :				List[Any]      =       CustomTokenizerFast.from_pretrained(UpperCamelCase__		)

      tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,						use_auth_token=self._token		)

      lowerCAmelCase_      :				str      =       AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'   ,						trust_remote_code=UpperCamelCase__		)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,						"""CustomTokenizerFast"""		)
      lowerCAmelCase_      :				Any      =       AutoTokenizer.from_pretrained(
          F'{USER}/test-dynamic-tokenizer'   ,						use_fast=UpperCamelCase__   ,						trust_remote_code=UpperCamelCase__		)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,						"""CustomTokenizer"""		)







class        __snake_case	(       unittest.TestCase):







    """simple docstring"""



    def __lowercase       (       self   :  Dict		)       ->  Union[str, Any]:
      lowerCAmelCase_      :				List[Any]      =       Trie()
      trie.add("""Hello 友達"""		)
      self.assertEqual(trie.data   ,						{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}		)
      trie.add("""Hello"""		)
      trie.data
      self.assertEqual(trie.data   ,						{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}		)

    def __lowercase       (       self   :  List[str]		)       ->  int:
      lowerCAmelCase_      :				Tuple      =       Trie()
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""		)   ,						["""[CLS] This is a extra_id_100"""]		)
      trie.add("""[CLS]"""		)
      trie.add("""extra_id_1"""		)
      trie.add("""extra_id_100"""		)
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""		)   ,						["""[CLS]""", """ This is a """, """extra_id_100"""]		)

    def __lowercase       (       self   :  str		)       ->  int:
      lowerCAmelCase_      :				str      =       Trie()
      trie.add("""A"""		)
      self.assertEqual(trie.split("""ABC"""		)   ,						["""A""", """BC"""]		)
      self.assertEqual(trie.split("""BCA"""		)   ,						["""BC""", """A"""]		)

    def __lowercase       (       self   :  Optional[Any]		)       ->  Dict:
      lowerCAmelCase_      :				Any      =       Trie()
      trie.add("""TOKEN]"""		)
      trie.add("""[SPECIAL_TOKEN]"""		)
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""		)   ,						["""This is something """, """[SPECIAL_TOKEN]"""]		)

    def __lowercase       (       self   :  Union[str, Any]		)       ->  Optional[int]:
      lowerCAmelCase_      :				Tuple      =       Trie()
      trie.add("""A"""		)
      trie.add("""P"""		)
      trie.add("""[SPECIAL_TOKEN]"""		)
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""		)   ,						["""This is something """, """[SPECIAL_TOKEN]"""]		)

    def __lowercase       (       self   :  Optional[int]		)       ->  Tuple:
      lowerCAmelCase_      :				Any      =       Trie()
      trie.add("""AB"""		)
      trie.add("""B"""		)
      trie.add("""C"""		)
      self.assertEqual(trie.split("""ABC"""		)   ,						["""AB""", """C"""]		)

    def __lowercase       (       self   :  List[str]		)       ->  Dict:
      lowerCAmelCase_      :				str      =       Trie()
      trie.add("""ABC"""		)
      trie.add("""B"""		)
      trie.add("""CD"""		)
      self.assertEqual(trie.split("""ABCD"""		)   ,						["""ABC""", """D"""]		)





    def __lowercase       (       self   :  str		)       ->  List[str]:
      lowerCAmelCase_      :				str      =       Trie()
      lowerCAmelCase_      :				List[Any]      =       trie.cut_text("""ABC"""   ,						[0, 0, 2, 1, 2, 3]		)
      self.assertEqual(UpperCamelCase__   ,						["""AB""", """C"""]		)








'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__A :  Any    							=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A :  int    							=				{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A :  List[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": 512,
}

__A :  Optional[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_INIT_CONFIGURATION
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= LxmertTokenizer

    def __init__(       self   :  Dict   ,						lowerCamelCase   :  Tuple=None   ,						lowerCamelCase   :  List[str]=None   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  List[Any]="[UNK]"   ,						lowerCamelCase   :  Tuple="[SEP]"   ,						lowerCamelCase   :  str="[PAD]"   ,						lowerCamelCase   :  Any="[CLS]"   ,						lowerCamelCase   :  str="[MASK]"   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  Union[str, Any]=None   ,						**lowerCamelCase   :  Optional[int]   ,						)       ->  Optional[Any]:
      super().__init__(
          lowerCamelCase   ,						tokenizer_file=lowerCamelCase   ,						do_lower_case=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						cls_token=lowerCamelCase   ,						mask_token=lowerCamelCase   ,						tokenize_chinese_chars=lowerCamelCase   ,						strip_accents=lowerCamelCase   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				Optional[Any]      =       json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
      if (
          normalizer_state.get("""lowercase"""   ,						lowerCamelCase		) != do_lower_case
          or normalizer_state.get("""strip_accents"""   ,						lowerCamelCase		) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""   ,						lowerCamelCase		) != tokenize_chinese_chars
      ):
        lowerCAmelCase_      :				Tuple      =       getattr(lowerCamelCase   ,						normalizer_state.pop("""type"""		)		)
        lowerCAmelCase_      :				Optional[Any]      =       do_lower_case
        lowerCAmelCase_      :				Dict      =       strip_accents
        lowerCAmelCase_      :				List[Any]      =       tokenize_chinese_chars
        lowerCAmelCase_      :				List[Any]      =       normalizer_class(**lowerCamelCase		)

      lowerCAmelCase_      :				Tuple      =       do_lower_case

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[Any]=None		)       ->  int:
      lowerCAmelCase_      :				Any      =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output

    def __lowercase       (       self   :  str   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:
      lowerCAmelCase_      :				str      =       [self.sep_token_id]
      lowerCAmelCase_      :				Optional[int]      =       [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
      return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      lowerCAmelCase_      :				int      =       self._tokenizer.model.save(lowerCamelCase   ,						name=lowerCamelCase		)
      return tuple(lowerCamelCase		)



