



import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case						:Any						=					logging.get_logger(__name__)

__snake_case						:str						=					{
    '''vocab_file''': '''vocab.txt''',
    '''merges_file''': '''bpe.codes''',
}

__snake_case						:List[Any]						=					{
    '''vocab_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt''',
    },
    '''merges_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes''',
    },
}

__snake_case						:List[Any]						=					{
    '''vinai/phobert-base''': 256,
    '''vinai/phobert-large''': 256,
}





def 					__snake_case     (      _UpperCAmelCase    ):
    __a							=    set()
    __a							=    word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)    )
        __a							=    char

    __a							=    set(_UpperCAmelCase    )
    return pairs

class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   Any						 =						VOCAB_FILES_NAMES
       UpperCamelCase__  :   Optional[Any]						 =						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :   Tuple						 =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



       def __init__(		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Any					,		__SCREAMING_SNAKE_CASE							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]="<s>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="</s>"					,		__SCREAMING_SNAKE_CASE							:					List[str]="</s>"					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]="<s>"					,		__SCREAMING_SNAKE_CASE							:					Dict="<unk>"					,		__SCREAMING_SNAKE_CASE							:					str="<pad>"					,		__SCREAMING_SNAKE_CASE							:					List[str]="<mask>"					,		**__SCREAMING_SNAKE_CASE							:					Dict					,		):


           '''simple docstring'''

           super().__init__(
               bos_token=__SCREAMING_SNAKE_CASE					,		eos_token=__SCREAMING_SNAKE_CASE					,		unk_token=__SCREAMING_SNAKE_CASE					,		sep_token=__SCREAMING_SNAKE_CASE					,		cls_token=__SCREAMING_SNAKE_CASE					,		pad_token=__SCREAMING_SNAKE_CASE					,		mask_token=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)

           __a							=    vocab_file
           __a							=    merges_file

           __a							=    {}
           __a							=    0
           __a							=    1
           __a							=    2
           __a							=    3

           self.add_from_file(__SCREAMING_SNAKE_CASE)

           __a							=    {v: k for k, v in self.encoder.items()}

           with open(__SCREAMING_SNAKE_CASE					,		encoding='''utf-8''') as merges_handle:
               __a							=    merges_handle.read().split('''\n''')[:-1]
           __a							=    [tuple(merge.split()[:-1]) for merge in merges]
           __a							=    dict(zip(__SCREAMING_SNAKE_CASE					,		range(len(__SCREAMING_SNAKE_CASE))))
           __a							=    {}



       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


           '''simple docstring'''

           if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           __a							=    [self.cls_token_id]
           __a							=    [self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None					,		__SCREAMING_SNAKE_CASE							:					bool = False):


           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__SCREAMING_SNAKE_CASE					,		token_ids_a=__SCREAMING_SNAKE_CASE					,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

           if token_ids_a is None:
               return [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]
           return [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]



       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


           '''simple docstring'''

           __a							=    [self.sep_token_id]
           __a							=    [self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]



       @property
       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           return len(self.encoder)



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           return dict(self.encoder					,		**self.added_tokens_encoder)



       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Dict):


           '''simple docstring'''

           if token in self.cache:
               return self.cache[token]
           __a							=    tuple(__SCREAMING_SNAKE_CASE)
           __a							=    tuple(list(word[:-1]) + [word[-1] + '''</w>'''])
           __a							=    get_pairs(__SCREAMING_SNAKE_CASE)

           if not pairs:
               return token

           while True:
               __a							=    min(__SCREAMING_SNAKE_CASE					,		key=lambda __SCREAMING_SNAKE_CASE: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE					,		float('''inf''')))
               if bigram not in self.bpe_ranks:
                   break
               __a	,       __a							=    bigram
               __a							=    []
               __a							=    0
               while i < len(__SCREAMING_SNAKE_CASE):
                   try:
                       __a							=    word.index(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)
                   except ValueError:
                       new_word.extend(word[i:])
                       break
                   else:
                       new_word.extend(word[i:j])
                       __a							=    j

                   if word[i] == first and i < len(__SCREAMING_SNAKE_CASE) - 1 and word[i + 1] == second:
                       new_word.append(first + second)
                       i += 2
                   else:
                       new_word.append(word[i])
                       i += 1
               __a							=    tuple(__SCREAMING_SNAKE_CASE)
               __a							=    new_word
               if len(__SCREAMING_SNAKE_CASE) == 1:
                   break
               else:
                   __a							=    get_pairs(__SCREAMING_SNAKE_CASE)
           __a							=    '''@@ '''.join(__SCREAMING_SNAKE_CASE)
           __a							=    word[:-4]
           __a							=    word
           return word



       def 		_lowerCamelCase      (		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           __a							=    []

           __a							=    re.findall(r'''\S+\n?'''					,		__SCREAMING_SNAKE_CASE)

           for token in words:
               split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE).split(''' ''')))
           return split_tokens



       def 		_lowerCamelCase      (		self							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					Tuple):


           '''simple docstring'''

           return self.encoder.get(__SCREAMING_SNAKE_CASE					,		self.encoder.get(self.unk_token))



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					str):


           '''simple docstring'''

           return self.decoder.get(__SCREAMING_SNAKE_CASE					,		self.unk_token)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]):


           '''simple docstring'''

           __a							=    ''' '''.join(__SCREAMING_SNAKE_CASE).replace('''@@ '''					,		'''''').strip()
           return out_string



       def 		_lowerCamelCase      (		self							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Optional[str] = None):


           '''simple docstring'''

           if not os.path.isdir(__SCREAMING_SNAKE_CASE):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
           __a							=    os.path.join(
               __SCREAMING_SNAKE_CASE					,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
           __a							=    os.path.join(
               __SCREAMING_SNAKE_CASE					,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE):
               copyfile(self.vocab_file					,		__SCREAMING_SNAKE_CASE)

           if os.path.abspath(self.merges_file) != os.path.abspath(__SCREAMING_SNAKE_CASE):
               copyfile(self.merges_file					,		__SCREAMING_SNAKE_CASE)

           return out_vocab_file, out_merge_file





       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
               try:
                   with open(__SCREAMING_SNAKE_CASE					,		'''r'''					,		encoding='''utf-8''') as fd:
                       self.add_from_file(__SCREAMING_SNAKE_CASE)
               except FileNotFoundError as fnfe:
                   raise fnfe
               except UnicodeError:
                   raise Exception(F'Incorrect encoding detected in {f}, please rebuild the dataset')
               return

           __a							=    f.readlines()
           for lineTmp in lines:
               __a							=    lineTmp.strip()
               __a							=    line.rfind(''' ''')
               if idx == -1:
                   raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\'''')
               __a							=    line[:idx]
               __a							=    len(self.encoder)










import os
import warnings
from typing import List, Optional

from ...tokenization_utils_base import BatchEncoding
from ...utils import logging
from .configuration_rag import RagConfig


__snake_case						:int						=					logging.get_logger(__name__)

class    _A	:



       def __init__(		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					Any					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           __a							=    question_encoder
           __a							=    generator
           __a							=    self.question_encoder



       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           if os.path.isfile(__SCREAMING_SNAKE_CASE):
               raise ValueError(F'Provided path ({save_directory}) should be a directory, not a file')
           os.makedirs(__SCREAMING_SNAKE_CASE					,		exist_ok=__SCREAMING_SNAKE_CASE)
           __a							=    os.path.join(__SCREAMING_SNAKE_CASE					,		'''question_encoder_tokenizer''')
           __a							=    os.path.join(__SCREAMING_SNAKE_CASE					,		'''generator_tokenizer''')
           self.question_encoder.save_pretrained(__SCREAMING_SNAKE_CASE)
           self.generator.save_pretrained(__SCREAMING_SNAKE_CASE)



       @classmethod
       def 		_lowerCamelCase      (		cls							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Any					,		**__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           from ..auto.tokenization_auto import AutoTokenizer

           __a							=    kwargs.pop('''config'''					,		__SCREAMING_SNAKE_CASE)

           if config is None:
               __a							=    RagConfig.from_pretrained(__SCREAMING_SNAKE_CASE)

           __a							=    AutoTokenizer.from_pretrained(
               __SCREAMING_SNAKE_CASE					,		config=config.question_encoder					,		subfolder='''question_encoder_tokenizer''')
           __a							=    AutoTokenizer.from_pretrained(
               __SCREAMING_SNAKE_CASE					,		config=config.generator					,		subfolder='''generator_tokenizer''')
           return cls(question_encoder=__SCREAMING_SNAKE_CASE					,		generator=__SCREAMING_SNAKE_CASE)



       def __call__(		self							:					Dict					,		*__SCREAMING_SNAKE_CASE							:					List[Any]					,		**__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           return self.current_tokenizer(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		*__SCREAMING_SNAKE_CASE							:					Dict					,		**__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           return self.generator.batch_decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		*__SCREAMING_SNAKE_CASE							:					List[str]					,		**__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           return self.generator.decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           __a							=    self.question_encoder



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    self.generator





       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[str]] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[int] = None					,		__SCREAMING_SNAKE_CASE							:					Optional[int] = None					,		__SCREAMING_SNAKE_CASE							:					str = "longest"					,		__SCREAMING_SNAKE_CASE							:					str = None					,		__SCREAMING_SNAKE_CASE							:					bool = True					,		**__SCREAMING_SNAKE_CASE							:					List[str]					,		):


           '''simple docstring'''

           warnings.warn(
               '''`prepare_seq2seq_batch` is deprecated and will be removed in version 5 of 🤗 Transformers. Use the '''
               '''regular `__call__` method to prepare your inputs and the tokenizer under the `with_target_tokenizer` '''
               '''context manager to prepare your targets. See the documentation of your specific tokenizer for more '''
               '''details'''					,		__SCREAMING_SNAKE_CASE					,		)
           if max_length is None:
               __a							=    self.current_tokenizer.model_max_length
           __a							=    self(
               __SCREAMING_SNAKE_CASE					,		add_special_tokens=__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		max_length=__SCREAMING_SNAKE_CASE					,		padding=__SCREAMING_SNAKE_CASE					,		truncation=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)
           if tgt_texts is None:
               return model_inputs
           # Process tgt_texts
           if max_target_length is None:
               __a							=    self.current_tokenizer.model_max_length
           __a							=    self(
               text_target=__SCREAMING_SNAKE_CASE					,		add_special_tokens=__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		padding=__SCREAMING_SNAKE_CASE					,		max_length=__SCREAMING_SNAKE_CASE					,		truncation=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)
           __a							=    labels['''input_ids''']
           return model_inputs





