

'''simple docstring'''







import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase						    =      get_tests_dir('''fixtures/test_sentencepiece.model''')


@require_sentencepiece
@require_tokenizers
class    A__ ( _snake_case  ,  unittest.TestCase     ):
    lowercase		   =						XLNetTokenizer
    lowercase		   =						XLNetTokenizerFast
    lowercase		   =						True
    lowercase		   =						True



    def    snake_case_     (   self    )  -> Dict:





           '''simple docstring'''
           super().setUp()

           # We have a SentencePiece fixture for testing
           A_		   = XLNetTokenizer(UpperCamelCase__						,			keep_accents=UpperCamelCase__    )
           tokenizer.sanitize_special_tokens()
           tokenizer.save_pretrained(self.tmpdirname    )



    def    snake_case_     (   self    )  -> Any:





           '''simple docstring'''
           A_		   = """<s>"""
           A_		   = 1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__    )						,			UpperCamelCase__    )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__    )						,			UpperCamelCase__    )



    def    snake_case_     (   self    )  -> Any:





           '''simple docstring'''
           A_		   = list(self.get_tokenizer().get_vocab().keys()    )

           self.assertEqual(vocab_keys[0]						,			"""<unk>"""    )
           self.assertEqual(vocab_keys[1]						,			"""<s>"""    )
           self.assertEqual(vocab_keys[-1]						,			"""<eod>"""    )
           self.assertEqual(len(UpperCamelCase__    )						,			1006    )



    def    snake_case_     (   self    )  -> Tuple:





           '''simple docstring'''
           self.assertEqual(self.get_tokenizer().vocab_size						,			1000    )



    def    snake_case_     (   self    )  -> List[str]:





           '''simple docstring'''
           A_		   = XLNetTokenizer(UpperCamelCase__						,			keep_accents=UpperCamelCase__    )

           A_		   = tokenizer.tokenize("""This is a test"""    )
           self.assertListEqual(UpperCamelCase__						,			["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__    )						,			[285, 46, 10, 170, 382]    )

           A_		   = tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
           self.assertListEqual(
               UpperCamelCase__						,			[
                   SPIECE_UNDERLINE + """I""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """9""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """s""",
                   """é""",
                   """.""",
               ]						,			)
           A_		   = tokenizer.convert_tokens_to_ids(UpperCamelCase__    )
           self.assertListEqual(UpperCamelCase__						,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]    )

           A_		   = tokenizer.convert_ids_to_tokens(UpperCamelCase__    )
           self.assertListEqual(
               UpperCamelCase__						,			[
                   SPIECE_UNDERLINE + """I""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """<unk>""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """s""",
                   """<unk>""",
                   """.""",
               ]						,			)



    def    snake_case_     (   self    )  -> Optional[Any]:





           '''simple docstring'''
           A_		   = XLNetTokenizer(UpperCamelCase__						,			do_lower_case=UpperCamelCase__    )
           A_		   = tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
           self.assertListEqual(
               UpperCamelCase__						,			[
                   SPIECE_UNDERLINE + """""",
                   """i""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """9""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """se""",
                   """.""",
               ]						,			)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,			["""▁he""", """ll""", """o"""]    )



    def    snake_case_     (   self    )  -> int:





           '''simple docstring'''
           A_		   = XLNetTokenizer(UpperCamelCase__						,			do_lower_case=UpperCamelCase__    )
           A_		   = tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
           self.assertListEqual(
               UpperCamelCase__						,			[
                   SPIECE_UNDERLINE + """I""",
                   SPIECE_UNDERLINE + """was""",
                   SPIECE_UNDERLINE + """b""",
                   """or""",
                   """n""",
                   SPIECE_UNDERLINE + """in""",
                   SPIECE_UNDERLINE + """""",
                   """9""",
                   """2""",
                   """0""",
                   """0""",
                   """0""",
                   """,""",
                   SPIECE_UNDERLINE + """and""",
                   SPIECE_UNDERLINE + """this""",
                   SPIECE_UNDERLINE + """is""",
                   SPIECE_UNDERLINE + """f""",
                   """al""",
                   """se""",
                   """.""",
               ]						,			)



    @slow
    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           A_		   = XLNetTokenizer.from_pretrained("""xlnet-base-cased"""    )

           A_		   = tokenizer.encode("""sequence builders"""						,			add_special_tokens=UpperCamelCase__    )
           A_		   = tokenizer.encode("""multi-sequence build"""						,			add_special_tokens=UpperCamelCase__    )

           A_		   = tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    )
           A_		   = tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						,			UpperCamelCase__    )

           assert encoded_sentence == text + [4, 3]
           assert encoded_pair == text + [4] + text_a + [4, 3]







    @slow
    def    snake_case_     (   self    )  -> Optional[int]:





           '''simple docstring'''
           # fmt: off
           A_		   = {"""input_ids""": [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase__						,			model_name="""xlnet-base-cased"""						,			revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""						,			)



'''simple docstring'''







import json
import logging
import os
import socket

import git
import numpy as np
import torch


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - PID: %(process)d -  %(message)s''',
    datefmt='''%m/%d/%Y %H:%M:%S''',
    level=logging.INFO,
)
__lowerCamelCase						    =      logging.getLogger(__name__)
def  UpperCAmelCase__      ( UpperCAmelCase__			)     ->     int:
       A_		   = git.Repo(search_parent_directories=UpperCAmelCase__			)
       A_		   = {
           """repo_id""": str(UpperCAmelCase__			),
           """repo_sha""": str(repo.head.object.hexsha			),
           """repo_branch""": str(repo.active_branch			),
       }

       with open(os.path.join(UpperCAmelCase__,    """git_log.json"""			),    """w"""			) as f:
              json.dump(UpperCAmelCase__,    UpperCAmelCase__,    indent=4			)
def  UpperCAmelCase__      ( UpperCAmelCase__			)     ->     Dict:
       if params.n_gpu <= 0:
              A_		   = 0
              A_		   = -1
              A_		   = True
              A_		   = False
              return

       assert torch.cuda.is_available()

       logger.info("""Initializing GPUs"""			)
       if params.n_gpu > 1:
              assert params.local_rank != -1

              A_		   = int(os.environ["""WORLD_SIZE"""]			)
              A_		   = int(os.environ["""N_GPU_NODE"""]			)
              A_		   = int(os.environ["""RANK"""]			)

              # number of nodes / node ID
              A_		   = params.world_size // params.n_gpu_per_node
              A_		   = params.global_rank // params.n_gpu_per_node
              A_		   = True

              assert params.n_nodes == int(os.environ["""N_NODES"""]			)
              assert params.node_id == int(os.environ["""NODE_RANK"""]			)

       # local job (single GPU)
       else:
              assert params.local_rank == -1

              A_		   = 1
              A_		   = 0
              A_		   = 0
              A_		   = 0
              A_		   = 1
              A_		   = 1
              A_		   = False

       # sanity checks
       assert params.n_nodes >= 1
       assert 0 <= params.node_id < params.n_nodes
       assert 0 <= params.local_rank <= params.global_rank < params.world_size
       assert params.world_size == params.n_nodes * params.n_gpu_per_node

       # define whether this is the master process / if we are in multi-node distributed mode
       A_		   = params.node_id == 0 and params.local_rank == 0
       A_		   = params.n_nodes > 1

       # summary
       A_		   = F'''--- Global rank: {params.global_rank} - '''
       logger.info(PREFIX + """Number of nodes: %i""" % params.n_nodes			)
       logger.info(PREFIX + """Node ID        : %i""" % params.node_id			)
       logger.info(PREFIX + """Local rank     : %i""" % params.local_rank			)
       logger.info(PREFIX + """World size     : %i""" % params.world_size			)
       logger.info(PREFIX + """GPUs per node  : %i""" % params.n_gpu_per_node			)
       logger.info(PREFIX + """Master         : %s""" % str(params.is_master			)			)
       logger.info(PREFIX + """Multi-node     : %s""" % str(params.multi_node			)			)
       logger.info(PREFIX + """Multi-GPU      : %s""" % str(params.multi_gpu			)			)
       logger.info(PREFIX + """Hostname       : %s""" % socket.gethostname()			)

       # set GPU device
       torch.cuda.set_device(params.local_rank			)

       # initialize multi-GPU
       if params.multi_gpu:
              logger.info("""Initializing PyTorch distributed"""			)
              torch.distributed.init_process_group(
                  init_method="""env://""",    backend="""nccl""",    )







def  UpperCAmelCase__      ( UpperCAmelCase__			)     ->     Tuple:
       np.random.seed(args.seed			)
       torch.manual_seed(args.seed			)
       if args.n_gpu > 0:
              torch.cuda.manual_seed_all(args.seed			)
