




'''simple docstring'''





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast

class        _a    (						unittest.TestCase							):






     def     _lowercase     (			self   )							->      Tuple:
          _snake_case							=				0






     @slow
     def     _lowercase     (			self   )							->      Tuple:
          for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(BertTokenizer, BertTokenizerFast)   )
               self.assertGreater(len(_SCREAMING_SNAKE_CASE   ) ,0   )

          for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(GPTaTokenizer, GPTaTokenizerFast)   )
               self.assertGreater(len(_SCREAMING_SNAKE_CASE   ) ,0   )






     def     _lowercase     (			self   )							->      Tuple:
          _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(BertTokenizer, BertTokenizerFast)   )
          self.assertEqual(tokenizer.vocab_size ,12   )






     def     _lowercase     (			self   )							->      Tuple:
          _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(RobertaTokenizer, RobertaTokenizerFast)   )
          self.assertEqual(tokenizer.vocab_size ,20   )






     def     _lowercase     (			self   )							->      int:
          _snake_case							=				AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )
          # Check that tokenizer_type ≠ model_type
          _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,config=_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(BertTokenizer, BertTokenizerFast)   )
          self.assertEqual(tokenizer.vocab_size ,12   )






     def     _lowercase     (			self   )							->      Optional[Any]:
          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("./tests/fixtures/vocab.txt" ,os.path.join(_SCREAMING_SNAKE_CASE ,"vocab.txt"   )   )

               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,tokenizer_type="bert" ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("./tests/fixtures/vocab.json" ,os.path.join(_SCREAMING_SNAKE_CASE ,"vocab.json"   )   )
               shutil.copy("./tests/fixtures/merges.txt" ,os.path.join(_SCREAMING_SNAKE_CASE ,"merges.txt"   )   )

               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,tokenizer_type="gpt2" ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      str:
          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("./tests/fixtures/vocab.txt" ,os.path.join(_SCREAMING_SNAKE_CASE ,"vocab.txt"   )   )

               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,tokenizer_type="bert"   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("./tests/fixtures/vocab.json" ,os.path.join(_SCREAMING_SNAKE_CASE ,"vocab.json"   )   )
               shutil.copy("./tests/fixtures/merges.txt" ,os.path.join(_SCREAMING_SNAKE_CASE ,"merges.txt"   )   )

               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,tokenizer_type="gpt2"   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      int:
          with pytest.raises(_SCREAMING_SNAKE_CASE   ):
               AutoTokenizer.from_pretrained("./" ,tokenizer_type="xxx"   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      int:
          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               _snake_case							=				tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"   )
               self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(BertTokenizer, BertTokenizerFast)   )

               if isinstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   ):
                    self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,_SCREAMING_SNAKE_CASE   )
               else:
                    self.assertEqual(tokenizer.do_lower_case ,_SCREAMING_SNAKE_CASE   )

               self.assertEqual(tokenizer.model_max_length ,512   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      Optional[Any]:
          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   _SCREAMING_SNAKE_CASE ,"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier" ,):
                    _snake_case							=				tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"   )






     def     _lowercase     (			self   )							->      List[Any]:
          # tests: https://github.com/huggingface/transformers/pull/13251
          # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
          # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
          _snake_case							=				TOKENIZER_MAPPING.values()
          _snake_case							=				[]

          for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                    tokenizer_names.append(slow_tok.__name__   )

               if fast_tok is not None:
                    tokenizer_names.append(fast_tok.__name__   )

          for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(_SCREAMING_SNAKE_CASE   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      Any:
          self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased" ,use_fast=_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"   ) ,_SCREAMING_SNAKE_CASE   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      str:
          _snake_case							=				AutoTokenizer.from_pretrained("distilbert-base-uncased" ,do_lower_case=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				"Hello, world. How are you?"
          _snake_case							=				tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          self.assertEqual("[UNK]" ,tokens[0]   )

          _snake_case							=				AutoTokenizer.from_pretrained("microsoft/mpnet-base" ,do_lower_case=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				tokenizer.tokenize(_SCREAMING_SNAKE_CASE   )
          self.assertEqual("[UNK]" ,tokens[0]   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      List[Any]:
          _snake_case							=				AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"   )
          self.assertEqual(type(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )
          self.assertEqual(tokenizer.model_max_length ,512   )
          self.assertEqual(tokenizer.vocab_size ,30_000   )
          self.assertEqual(tokenizer.unk_token ,"[UNK]"   )
          self.assertEqual(tokenizer.padding_side ,"right"   )
          self.assertEqual(tokenizer.truncation_side ,"right"   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,(BertTokenizer, BertTokenizerFast)   )
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )

          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,tokenizer.__class__   )
          self.assertEqual(tokenizera.vocab_size ,12   )






     def     _lowercase     (			self   )							->      List[Any]:
          _snake_case							=				AutoTokenizer.from_pretrained("ctrl"   )
          # There is no fast CTRL so this always gives us a slow tokenizer.
          self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Tuple:
          # Check we can load the tokenizer config of an online model.
          _snake_case							=				get_tokenizer_config("bert-base-cased"   )
          _snake_case							=				config.pop("_commit_hash" ,_SCREAMING_SNAKE_CASE   )
          # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
          self.assertEqual(_SCREAMING_SNAKE_CASE ,{"do_lower_case": False}   )

          # This model does not have a tokenizer_config so we get back an empty dict.
          _snake_case							=				get_tokenizer_config(_SCREAMING_SNAKE_CASE   )
          self.assertDictEqual(_SCREAMING_SNAKE_CASE ,{}   )

          # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
          _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
               _snake_case							=				get_tokenizer_config(_SCREAMING_SNAKE_CASE   )

          # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
          self.assertEqual(config["tokenizer_class"] ,"BertTokenizer"   )






     def     _lowercase     (			self   )							->      Optional[Any]:
          try:
               AutoConfig.register("custom" ,_SCREAMING_SNAKE_CASE   )

               AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_SCREAMING_SNAKE_CASE   ):
                    AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE   )

               _snake_case							=				CustomTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )

                    _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
                    self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]






     @require_tokenizers
     def     _lowercase     (			self   )							->      Union[str, Any]:
          try:
               AutoConfig.register("custom" ,_SCREAMING_SNAKE_CASE   )

               # Can register in two steps
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None)   )
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast)   )

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   _SCREAMING_SNAKE_CASE ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast)   )

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_SCREAMING_SNAKE_CASE   ):
                    AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE   )

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                    _snake_case							=				BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE   )
                    bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
                    _snake_case							=				CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE   )

               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )

                    _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
                    self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

                    _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
                    self.assertIsInstance(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def     _lowercase     (			self   )							->      Union[str, Any]:
          # If remote code is not set, we will time out when asking whether to load the model.
          with self.assertRaises(_SCREAMING_SNAKE_CASE   ):
               _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"   )
          # If remote code is disabled, we can't load this config.
          with self.assertRaises(_SCREAMING_SNAKE_CASE   ):
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE   )

          _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE   )
          self.assertTrue(tokenizer.special_attribute_present   )
          # Test tokenizer can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,trust_remote_code=_SCREAMING_SNAKE_CASE   )
          self.assertTrue(reloaded_tokenizer.special_attribute_present   )

          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizerFast"   )
               self.assertEqual(reloaded_tokenizer.__class__.__name__ ,"NewTokenizerFast"   )

               # Test we can also load the slow version
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertTrue(tokenizer.special_attribute_present   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
                    _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,trust_remote_code=_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(reloaded_tokenizer.__class__.__name__ ,"NewTokenizer"   )
               self.assertTrue(reloaded_tokenizer.special_attribute_present   )
          else:
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
               self.assertEqual(reloaded_tokenizer.__class__.__name__ ,"NewTokenizer"   )






     @require_tokenizers
     def     _lowercase     (			self   )							->      Optional[int]:

          class        _a    (						__lowerCAmelCase							):
               SCREAMING_SNAKE_CASE_							:   Any       =						False







          class        _a    (						__lowerCAmelCase							):
               SCREAMING_SNAKE_CASE_							:   Dict       =						NewTokenizer
               SCREAMING_SNAKE_CASE_							:   List[str]       =						False

          try:
               AutoConfig.register("custom" ,_SCREAMING_SNAKE_CASE   )
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE   )
               # If remote code is not set, the default is to use local
               _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizerFast"   )
               self.assertFalse(tokenizer.special_attribute_present   )
               _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer" ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
               self.assertFalse(tokenizer.special_attribute_present   )

               # If remote code is disabled, we load the local one.
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizerFast"   )
               self.assertFalse(tokenizer.special_attribute_present   )
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
               self.assertFalse(tokenizer.special_attribute_present   )

               # If remote is enabled, we load from the Hub
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizerFast"   )
               self.assertTrue(tokenizer.special_attribute_present   )
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer" ,trust_remote_code=_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
               self.assertTrue(tokenizer.special_attribute_present   )

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]






     def     _lowercase     (			self   )							->      int:
          _snake_case							=				AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer_legacy" ,trust_remote_code=_SCREAMING_SNAKE_CASE   )
          self.assertTrue(tokenizer.special_attribute_present   )
          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizerFast"   )

               # Test we can also load the slow version
               _snake_case							=				AutoTokenizer.from_pretrained(
                   "hf-internal-testing/test_dynamic_tokenizer_legacy" ,trust_remote_code=_SCREAMING_SNAKE_CASE ,use_fast=_SCREAMING_SNAKE_CASE   )
               self.assertTrue(tokenizer.special_attribute_present   )
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )
          else:
               self.assertEqual(tokenizer.__class__.__name__ ,"NewTokenizer"   )






     def     _lowercase     (			self   )							->      List[str]:
          with self.assertRaisesRegex(
              _SCREAMING_SNAKE_CASE ,"bert-base is not a local folder and is not a valid model identifier"   ):
               _snake_case							=				AutoTokenizer.from_pretrained("bert-base"   )






     def     _lowercase     (			self   )							->      Tuple:
          with self.assertRaisesRegex(
              _SCREAMING_SNAKE_CASE ,r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"   ):
               _snake_case							=				AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE ,revision="aaaaaa"   )






     def     _lowercase     (			self   )							->      List[str]:
          # Make sure we have cached the tokenizer.
          _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"   )
          with RequestCounter() as counter:
               _snake_case							=				AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"   )
               self.assertEqual(counter.get_request_count ,0   )
               self.assertEqual(counter.head_request_count ,1   )
               self.assertEqual(counter.other_request_count ,0   )








'''simple docstring'''





import argparse
import gc
import json
import os
import re

import torch
from huggingface_hub import hf_hub_download

from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerFast, RwkvConfig
from transformers.modeling_utils import WEIGHTS_INDEX_NAME, shard_checkpoint


UpperCamelCase_						:      int       =						{
    '''169M''': 12,
    '''430M''': 24,
    '''1B5''': 24,
    '''3B''': 32,
    '''7B''': 32,
    '''14B''': 40,
}

UpperCamelCase_						:      str       =						{
    '''169M''': 768,
    '''430M''': 1024,
    '''1B5''': 2048,
    '''3B''': 2560,
    '''7B''': 4096,
    '''14B''': 5120,
}





def      __a     (       _UpperCamelCase:   str   )						->			Any:

     """simple docstring"""




     _snake_case							=				list(state_dict.keys()   )
     for name in state_dict_keys:
          _snake_case							=				state_dict.pop(_UpperCamelCase   )
          # emb -> embedding
          if name.startswith("emb."   ):
               _snake_case							=				name.replace("emb." ,						"embeddings."   )
          # ln_0 -> pre_ln (only present at block 0)
          if name.startswith("blocks.0.ln0"   ):
               _snake_case							=				name.replace("blocks.0.ln0" ,						"blocks.0.pre_ln"   )
          # att -> attention
          _snake_case							=				re.sub(r"blocks\.(\d+)\.att" ,						r"blocks.\1.attention" ,						_UpperCamelCase   )
          # ffn -> feed_forward
          _snake_case							=				re.sub(r"blocks\.(\d+)\.ffn" ,						r"blocks.\1.feed_forward" ,						_UpperCamelCase   )
          # time_mix_k -> time_mix_key and reshape
          if name.endswith(".time_mix_k"   ):
               _snake_case							=				name.replace(".time_mix_k" ,						".time_mix_key"   )
          # time_mix_v -> time_mix_value and reshape
          if name.endswith(".time_mix_v"   ):
               _snake_case							=				name.replace(".time_mix_v" ,						".time_mix_value"   )
          # time_mix_r -> time_mix_key and reshape
          if name.endswith(".time_mix_r"   ):
               _snake_case							=				name.replace(".time_mix_r" ,						".time_mix_receptance"   )

          if name != "head.weight":
               _snake_case							=				"rwkv." + name

          _snake_case							=				weight
     return state_dict





def      __a     (       _UpperCamelCase:   Any ,						_UpperCamelCase:   List[Any] ,						_UpperCamelCase:   List[Any] ,						_UpperCamelCase:   str=None ,						_UpperCamelCase:   Optional[Any]=None ,						_UpperCamelCase:   List[str]=False ,						_UpperCamelCase:   Dict=None   )						->			Dict:

     """simple docstring"""




     if tokenizer_file is None:
          print("No `--tokenizer_file` provided, we will use the default tokenizer."   )
          _snake_case							=				50_277
          _snake_case							=				AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b"   )
     else:
          _snake_case							=				PreTrainedTokenizerFast(tokenizer_file=_UpperCamelCase   )
          _snake_case							=				len(_UpperCamelCase   )
     tokenizer.save_pretrained(_UpperCamelCase   )

     # 2. Build the config
     _snake_case							=				list(NUM_HIDDEN_LAYERS_MAPPING.keys()   )
     if size is None:
          # Try to infer size from the checkpoint name
          for candidate in possible_sizes:
               if candidate in checkpoint_file:
                    _snake_case							=				candidate
                    break
          if size is None:
               raise ValueError("Could not infer the size, please provide it with the `--size` argument."   )
     if size not in possible_sizes:
          raise ValueError(F"""`size` should be one of {possible_sizes}, got {size}."""   )

     _snake_case							=				RwkvConfig(
         vocab_size=_UpperCamelCase ,						num_hidden_layers=NUM_HIDDEN_LAYERS_MAPPING[size] ,						hidden_size=HIDEN_SIZE_MAPPING[size] ,						)
     config.save_pretrained(_UpperCamelCase   )

     # 3. Download model file then convert state_dict
     _snake_case							=				hf_hub_download(_UpperCamelCase ,						_UpperCamelCase   )
     _snake_case							=				torch.load(_UpperCamelCase ,						map_location="cpu"   )
     _snake_case							=				convert_state_dict(_UpperCamelCase   )

     # 4. Split in shards and save
     _snake_case							, _snake_case							=				shard_checkpoint(_UpperCamelCase   )
     for shard_file, shard in shards.items():
          torch.save(_UpperCamelCase ,						os.path.join(_UpperCamelCase ,						_UpperCamelCase   )   )

     if index is not None:
          _snake_case							=				os.path.join(_UpperCamelCase ,						_UpperCamelCase   )
          # Save the index as well
          with open(_UpperCamelCase ,						"w" ,						encoding="utf-8"   ) as f:
               _snake_case							=				json.dumps(_UpperCamelCase ,						indent=2 ,						sort_keys=_UpperCamelCase   ) + "\n"
               f.write(_UpperCamelCase   )

          # 5. Clean up shards (for some reason the file PyTorch saves take the same space as the whole state_dict
          print(
              "Cleaning up shards. This may error with an OOM error, it this is the case don't worry you still have converted the model."   )
          _snake_case							=				list(shards.keys()   )

          del state_dict
          del shards
          gc.collect()

          for shard_file in shard_files:
               _snake_case							=				torch.load(os.path.join(_UpperCamelCase ,						_UpperCamelCase   )   )
               torch.save({k: v.cpu().clone() for k, v in state_dict.items()} ,						os.path.join(_UpperCamelCase ,						_UpperCamelCase   )   )

     del state_dict
     gc.collect()

     if push_to_hub:
          if model_name is None:
               raise ValueError("Please provide a `model_name` to push the model to the Hub."   )
          _snake_case							=				AutoModelForCausalLM.from_pretrained(_UpperCamelCase   )
          model.push_to_hub(_UpperCamelCase ,						max_shard_size="2GB"   )
          tokenizer.push_to_hub(_UpperCamelCase   )


if __name__ == "__main__":
   UpperCamelCase_						:      Any       =						argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '''--repo_id''', default=None, type=str, required=True, help='''Repo ID from which to pull the checkpoint.'''
   )
   parser.add_argument(
       '''--checkpoint_file''', default=None, type=str, required=True, help='''Name of the checkpoint file in the repo.'''
   )
   parser.add_argument(
       '''--output_dir''', default=None, type=str, required=True, help='''Where to save the converted model.'''
   )
   parser.add_argument(
       '''--tokenizer_file''',
       default=None,
       type=str,
       help='''Path to the tokenizer file to use (if not provided, only the model is converted).''',
   )
   parser.add_argument(
       '''--size''',
       default=None,
       type=str,
       help='''Size of the model. Will be inferred from the `checkpoint_file` if not passed.''',
   )
   parser.add_argument(
       '''--push_to_hub''',
       action='''store_true''',
       help='''Push to the Hub the converted model.''',
   )
   parser.add_argument(
       '''--model_name''',
       default=None,
       type=str,
       help='''Name of the pushed model on the Hub, including the username / organization.''',
   )

   UpperCamelCase_						:      int       =						parser.parse_args()
   convert_rmkv_checkpoint_to_hf_format(
       args.repo_id,
       args.checkpoint_file,
       args.output_dir,
       size=args.size,
       tokenizer_file=args.tokenizer_file,
       push_to_hub=args.push_to_hub,
       model_name=args.model_name,
   )


