
# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
lowercase__          =   abspath(join(dirname(__file__), 'src'))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='ignore', category=FutureWarning)




def        __UpperCamelCase					(    lowerCAmelCase__							:    Optional[int]  ):
 config.addinivalue_line(
     '''markers'''  ,       '''is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested'''  )
 config.addinivalue_line(
     '''markers'''  ,       '''is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested'''  )
 config.addinivalue_line('''markers'''  ,       '''is_pipeline_test: mark test to run only when pipelines are tested'''  )
 config.addinivalue_line('''markers'''  ,       '''is_staging_test: mark test to run only in the staging environment'''  )
 config.addinivalue_line('''markers'''  ,       '''accelerate_tests: mark test that require accelerate'''  )
 config.addinivalue_line('''markers'''  ,       '''tool_tests: mark the tool tests that are run on their specific schedule'''  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    Tuple  ):
 from transformers.testing_utils import pytest_addoption_shared

 pytest_addoption_shared(lowerCAmelCase__  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    Dict  ):
 from transformers.testing_utils import pytest_terminal_summary_main

 __a			:  List[Any]		    =  terminalreporter.config.getoption('''--make-reports'''  )
 if make_reports:
  pytest_terminal_summary_main(lowerCAmelCase__  ,       id=lowerCAmelCase__  )




def        __UpperCamelCase					(    lowerCAmelCase__							:    List[Any]  ,       lowerCAmelCase__							:    Optional[Any]  ):
 # If no tests are collected, pytest exists with code 5, which makes the CI fail.
 if exitstatus == 5:
  __a			:  List[Any]		    =  0


# Doctest custom flag to ignore output.
lowercase__          =   doctest.register_optionflag('IGNORE_RESULT')

lowercase__          =   doctest.OutputChecker



class 		UpperCamelCase__  (	__lowercase   ):
   def 				lowerCAmelCase      (self			: Any ,    snake_case_			: int ,    snake_case_			: Union[str, Any] ,    snake_case_			: List[Any]       ):
    if IGNORE_RESULT & optionflags:
     return True
    return OutputChecker.check_output(self ,    snake_case_ ,    snake_case_ ,    snake_case_       )





lowercase__          =   CustomOutputChecker
lowercase__          =   HfDoctestModule


lowercase__          =   HfDocTestParser


import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 		UpperCamelCase__  (	__lowercase   ,unittest.TestCase   ):
   _SCREAMING_SNAKE_CASE				:							Union[str, Any]     	=				PhobertTokenizer
   _SCREAMING_SNAKE_CASE				:							int     	=				False
   def 				lowerCAmelCase      (self			: Optional[int]       ):
    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    __a			:  Optional[int]		    =  ['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
    __a			:  Tuple		    =  dict(zip(snake_case_ ,    range(len(snake_case_       )       )       )       )
    __a			:  int		    =  ['''#version: 0.2''', '''l à</w>''']
    __a			:  List[Any]		    =  {'''unk_token''': '''<unk>'''}

    __a			:  Any		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''vocab_file''']       )
    __a			:  List[Any]		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''merges_file''']       )

    with open(self.vocab_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     for token in vocab_tokens:
      fp.write(f"{token} {vocab_tokens[token]}\n"       )
    with open(self.merges_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     fp.write('''\n'''.join(snake_case_       )       )
   def 				lowerCAmelCase      (self			: Union[str, Any] ,    **snake_case_			: List[str]       ):
    kwargs.update(self.special_tokens_map       )
    return PhobertTokenizer.from_pretrained(self.tmpdirname ,    **snake_case_       )
   def 				lowerCAmelCase      (self			: Any ,    snake_case_			: Dict       ):
    __a			:  Union[str, Any]		    =  '''Tôi là VinAI Research'''
    __a			:  int		    =  '''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
    return input_text, output_text







   def 				lowerCAmelCase      (self			: Optional[Any]       ):
    __a			:  Optional[int]		    =  PhobertTokenizer(self.vocab_file ,    self.merges_file ,    **self.special_tokens_map       )
    __a			:  Any		    =  '''Tôi là VinAI Research'''
    __a			:  Union[str, Any]		    =  '''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
    __a			:  List[str]		    =  tokenizer.tokenize(snake_case_       )
    print(snake_case_       )
    self.assertListEqual(snake_case_ ,    snake_case_       )

    __a			:  str		    =  tokens + [tokenizer.unk_token]

    __a			:  str		    =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_       ) ,    snake_case_       )
