


"""simple docstring"""



import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer







class _lowerCAmelCase    (				tf.keras.layers.Layer     ):







   def __init__(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						)     ->		int:

       '''simple docstring'''




       super().__init__()
       snake_case   :				Optional[int]				 =   pad_token_id
       snake_case   :				Any				 =   max_length
       snake_case   :				Tuple				 =   vocab
       snake_case   :				Optional[int]				 =   merges
       snake_case   :				List[str]				 =   BytePairTokenizer(UpperCamelCase__						,     UpperCamelCase__						,     sequence_length=UpperCamelCase__						)







   @classmethod
   def 			lowerCamelCase		(     cls						,     UpperCamelCase__						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Any:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   [" ".join(UpperCamelCase__						) for m in tokenizer.bpe_ranks.keys()]
       snake_case   :				Any				 =   tokenizer.get_vocab()
       return cls(UpperCamelCase__						,     UpperCamelCase__						,     *UpperCamelCase__						,     **UpperCamelCase__						)







   @classmethod
   def 			lowerCamelCase		(     cls						,     UpperCamelCase__						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   GPTaTokenizer.from_pretrained(UpperCamelCase__						,     *UpperCamelCase__						,     **UpperCamelCase__						)
       return cls.from_tokenizer(UpperCamelCase__						,     *UpperCamelCase__						,     **UpperCamelCase__						)







   @classmethod
   def 			lowerCamelCase		(     cls						,     UpperCamelCase__						)     ->		Optional[int]:

       '''simple docstring'''




       return cls(**UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       return {
           "vocab": self.vocab,
           "merges": self.merges,
           "max_length": self.max_length,
           "pad_token_id": self.pad_token_id,
       }







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   self.tf_tokenizer(UpperCamelCase__						)
       snake_case   :				Optional[Any]				 =   tf.ones_like(UpperCamelCase__						)

       if self.pad_token_id is not None:
           # pad the tokens up to max length
           snake_case   :				Optional[int]				 =   max_length if max_length is not None else self.max_length

           if max_length is not None:
               snake_case   :				Tuple				 =   pad_model_inputs(
                   UpperCamelCase__						,     max_seq_length=UpperCamelCase__						,     pad_value=self.pad_token_id						)

       return {"attention_mask": attention_mask, "input_ids": input_ids}









"""simple docstring"""



import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


__snake_case             =   logging.getLogger()







@unittest.skip('''Temporarily disable the doc tests.'''     )
@require_torch
@require_tf
@slow
class _lowerCAmelCase    (				unittest.TestCase     ):







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = True						,     )     ->		Tuple:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   [file for file in os.listdir(UpperCamelCase__						) if os.path.isfile(os.path.join(UpperCamelCase__						,     UpperCamelCase__						)						)]

       if identifier is not None:
           snake_case   :				Optional[Any]				 =   [file for file in files if identifier in file]

       if n_identifier is not None:
           if isinstance(UpperCamelCase__						,     UpperCamelCase__						):
               for n_ in n_identifier:
                   snake_case   :				str				 =   [file for file in files if n_ not in file]
           else:
               snake_case   :				str				 =   [file for file in files if n_identifier not in file]

       snake_case   :				Tuple				 =   ignore_files or []
       ignore_files.append("__init__.py"						)
       snake_case   :				Optional[Any]				 =   [file for file in files if file not in ignore_files]

       for file in files:
           # Open all files
           print("Testing"						,     UpperCamelCase__						)

           if only_modules:
               snake_case   :				str				 =   file.split("."						)[0]
               try:
                   snake_case   :				Optional[int]				 =   getattr(UpperCamelCase__						,     UpperCamelCase__						)
                   snake_case   :				str				 =   doctest.DocTestSuite(UpperCamelCase__						)
                   snake_case   :				Optional[Any]				 =   unittest.TextTestRunner().run(UpperCamelCase__						)
                   self.assertIs(len(result.failures						)						,     0						)
               except AttributeError:
                   logger.info(F'{module_identifier} is not a module.'						)
           else:
               snake_case   :				Tuple				 =   doctest.testfile(str(".." / directory / file						)						,     optionflags=doctest.ELLIPSIS						)
               self.assertIs(result.failed						,     0						)







   def 			lowerCamelCase		(     self						)     ->		str:

       '''simple docstring'''




       snake_case   :				Tuple				 =   Path("src/transformers"						)
       snake_case   :				List[Any]				 =   "modeling"
       snake_case   :				Dict				 =   [
           "modeling_ctrl.py",
           "modeling_tf_ctrl.py",
       ]
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						,     ignore_files=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "tokenization"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   Path("src/transformers"						)
       snake_case   :				Optional[Any]				 =   "configuration"
       self.analyze_directory(UpperCamelCase__						,     identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       snake_case   :				List[str]				 =   Path("src/transformers"						)
       snake_case   :				List[str]				 =   ["configuration", "modeling", "tokenization"]
       self.analyze_directory(UpperCamelCase__						,     n_identifier=UpperCamelCase__						)







   def 			lowerCamelCase		(     self						)     ->		List[Any]:

       '''simple docstring'''




       snake_case   :				Any				 =   Path("docs/source"						)
       snake_case   :				Tuple				 =   ["favicon.ico"]
       self.analyze_directory(UpperCamelCase__						,     ignore_files=UpperCamelCase__						,     only_modules=UpperCamelCase__						)





