
import os
import unittest

from huggingface_hub.utils import are_progress_bars_disabled

import transformers.models.bart.tokenization_bart
from transformers import logging
from transformers.testing_utils import CaptureLogger, mockenv, mockenv_context
from transformers.utils.logging import disable_progress_bar, enable_progress_bar



class 		UpperCamelCase__  (	unittest.TestCase   ):
   def 				lowerCAmelCase      (self			: Union[str, Any]       ):
    __a			:  Dict		    =  logging.get_logger()

    # the current default level is logging.WARNING
    __a			:  Any		    =  logging.get_verbosity()

    logging.set_verbosity_error()
    self.assertEqual(logger.getEffectiveLevel() ,    logging.get_verbosity()       )

    logging.set_verbosity_warning()
    self.assertEqual(logger.getEffectiveLevel() ,    logging.get_verbosity()       )

    logging.set_verbosity_info()
    self.assertEqual(logger.getEffectiveLevel() ,    logging.get_verbosity()       )

    logging.set_verbosity_debug()
    self.assertEqual(logger.getEffectiveLevel() ,    logging.get_verbosity()       )

    # restore to the original level
    logging.set_verbosity(snake_case_       )
   def 				lowerCAmelCase      (self			: int       ):
    __a			:  Dict		    =  logging.get_verbosity()

    __a			:  str		    =  logging.get_logger('''transformers.models.bart.tokenization_bart'''       )
    __a			:  str		    =  '''Testing 1, 2, 3'''

    # should be able to log warnings (if default settings weren't overridden by `pytest --log-level-all`)
    if level_origin <= logging.WARNING:
     with CaptureLogger(snake_case_       ) as cl:
      logger.warning(snake_case_       )
     self.assertEqual(cl.out ,    msg + '''\n'''       )

    # this is setting the level for all of `transformers.*` loggers
    logging.set_verbosity_error()

    # should not be able to log warnings
    with CaptureLogger(snake_case_       ) as cl:
     logger.warning(snake_case_       )
    self.assertEqual(cl.out ,    ''''''       )

    # should be able to log warnings again
    logging.set_verbosity_warning()
    with CaptureLogger(snake_case_       ) as cl:
     logger.warning(snake_case_       )
    self.assertEqual(cl.out ,    msg + '''\n'''       )

    # restore to the original level
    logging.set_verbosity(snake_case_       )
   @mockenv(TRANSFORMERS_VERBOSITY='''error'''       )
   def 				lowerCAmelCase      (self			: Optional[int]       ):
    # reset for the env var to take effect, next time some logger call is made
    transformers.utils.logging._reset_library_root_logger()
    # this action activates the env var
    __a			:  Optional[Any]		    =  logging.get_logger('''transformers.models.bart.tokenization_bart'''       )

    __a			:  List[str]		    =  os.getenv('''TRANSFORMERS_VERBOSITY''' ,    snake_case_       )
    __a			:  Union[str, Any]		    =  logging.log_levels[env_level_str]

    __a			:  int		    =  logging.get_verbosity()
    self.assertEqual(
        snake_case_ ,    snake_case_ ,    f"TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}" ,    )

    # restore to the original level
    __a			:  str		    =  ''''''
    transformers.utils.logging._reset_library_root_logger()
   @mockenv(TRANSFORMERS_VERBOSITY='''super-error'''       )
   def 				lowerCAmelCase      (self			: List[str]       ):
    # reset for the env var to take effect, next time some logger call is made
    transformers.utils.logging._reset_library_root_logger()
    __a			:  Optional[int]		    =  logging.logging.getLogger()
    with CaptureLogger(snake_case_       ) as cl:
     # this action activates the env var
     logging.get_logger('''transformers.models.bart.tokenization_bart'''       )
    self.assertIn('''Unknown option TRANSFORMERS_VERBOSITY=super-error''' ,    cl.out       )

    # no need to restore as nothing was changed







   def 				lowerCAmelCase      (self			: Union[str, Any]       ):
    # testing `logger.warning_advice()`
    transformers.utils.logging._reset_library_root_logger()

    __a			:  int		    =  logging.get_logger('''transformers.models.bart.tokenization_bart'''       )
    __a			:  Union[str, Any]		    =  '''Testing 1, 2, 3'''

    with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS='''1'''       ):
     # nothing should be logged as env var disables this method
     with CaptureLogger(snake_case_       ) as cl:
      logger.warning_advice(snake_case_       )
     self.assertEqual(cl.out ,    ''''''       )

    with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS=''''''       ):
     # should log normally as TRANSFORMERS_NO_ADVISORY_WARNINGS is unset
     with CaptureLogger(snake_case_       ) as cl:
      logger.warning_advice(snake_case_       )
     self.assertEqual(cl.out ,    msg + '''\n'''       )





def        __UpperCamelCase					(    ):
 disable_progress_bar()
 assert are_progress_bars_disabled()

 enable_progress_bar()
 assert not are_progress_bars_disabled()


import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 		UpperCamelCase__  (	__lowercase   ,unittest.TestCase   ):
   _SCREAMING_SNAKE_CASE				:							Union[str, Any]     	=				PhobertTokenizer
   _SCREAMING_SNAKE_CASE				:							int     	=				False
   def 				lowerCAmelCase      (self			: Optional[int]       ):
    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    __a			:  Optional[int]		    =  ['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
    __a			:  Tuple		    =  dict(zip(snake_case_ ,    range(len(snake_case_       )       )       )       )
    __a			:  int		    =  ['''#version: 0.2''', '''l à</w>''']
    __a			:  List[Any]		    =  {'''unk_token''': '''<unk>'''}

    __a			:  Any		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''vocab_file''']       )
    __a			:  List[Any]		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''merges_file''']       )

    with open(self.vocab_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     for token in vocab_tokens:
      fp.write(f"{token} {vocab_tokens[token]}\n"       )
    with open(self.merges_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     fp.write('''\n'''.join(snake_case_       )       )
   def 				lowerCAmelCase      (self			: Union[str, Any] ,    **snake_case_			: List[str]       ):
    kwargs.update(self.special_tokens_map       )
    return PhobertTokenizer.from_pretrained(self.tmpdirname ,    **snake_case_       )
   def 				lowerCAmelCase      (self			: Any ,    snake_case_			: Dict       ):
    __a			:  Union[str, Any]		    =  '''Tôi là VinAI Research'''
    __a			:  int		    =  '''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
    return input_text, output_text







   def 				lowerCAmelCase      (self			: Optional[Any]       ):
    __a			:  Optional[int]		    =  PhobertTokenizer(self.vocab_file ,    self.merges_file ,    **self.special_tokens_map       )
    __a			:  Any		    =  '''Tôi là VinAI Research'''
    __a			:  Union[str, Any]		    =  '''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
    __a			:  List[str]		    =  tokenizer.tokenize(snake_case_       )
    print(snake_case_       )
    self.assertListEqual(snake_case_ ,    snake_case_       )

    __a			:  str		    =  tokens + [tokenizer.unk_token]

    __a			:  str		    =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_       ) ,    snake_case_       )
