



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_:Dict					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Tuple					=						{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

SCREAMING_SNAKE_CASE_:Optional[int]					=						{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

SCREAMING_SNAKE_CASE_:Optional[Any]					=						{
    """ctrl""": 256,
}

SCREAMING_SNAKE_CASE_:List[Any]					=						{
    """Pregnancy""": 168_629,
    """Christianity""": 7_675,
    """Explain""": 106_423,
    """Fitness""": 63_440,
    """Saving""": 63_163,
    """Ask""": 27_171,
    """Ass""": 95_985,
    """Joke""": 163_509,
    """Questions""": 45_622,
    """Thoughts""": 49_605,
    """Retail""": 52_342,
    """Feminism""": 164_338,
    """Writing""": 11_992,
    """Atheism""": 192_263,
    """Netflix""": 48_616,
    """Computing""": 39_639,
    """Opinion""": 43_213,
    """Alone""": 44_967,
    """Funny""": 58_917,
    """Gaming""": 40_358,
    """Human""": 4_088,
    """India""": 1_331,
    """Joker""": 77_138,
    """Diet""": 36_206,
    """Legal""": 11_859,
    """Norman""": 4_939,
    """Tip""": 72_689,
    """Weight""": 52_343,
    """Movies""": 46_273,
    """Running""": 23_425,
    """Science""": 2_090,
    """Horror""": 37_793,
    """Confession""": 60_572,
    """Finance""": 12_250,
    """Politics""": 16_360,
    """Scary""": 191_985,
    """Support""": 12_654,
    """Technologies""": 32_516,
    """Teenage""": 66_160,
    """Event""": 32_769,
    """Learned""": 67_460,
    """Notion""": 182_770,
    """Wikipedia""": 37_583,
    """Books""": 6_665,
    """Extract""": 76_050,
    """Confessions""": 102_701,
    """Conspiracy""": 75_932,
    """Links""": 63_674,
    """Narcissus""": 150_425,
    """Relationship""": 54_766,
    """Relationships""": 134_796,
    """Reviews""": 41_671,
    """News""": 4_256,
    """Translation""": 26_820,
    """multilingual""": 128_406,
}




def      __UpperCamelCase     (  _lowerCAmelCase		)	->  Any:



    """simple docstring"""






    A			:	Union[str, Any]										=       set()
    A			:	str										=       word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)		)
        A			:	Union[str, Any]										=       char

    A			:	Optional[Any]										=       set(_lowerCAmelCase		)
    return pairs






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			Dict =				VOCAB_FILES_NAMES
    __lowerCamelCase      :			Dict =				PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase      :			Tuple =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase      :			Dict =				CONTROL_CODES






    def __init__(	self,		lowerCamelCase__,		lowerCamelCase__,		lowerCamelCase__="<unk>",		**lowerCamelCase__		):
        super().__init__(unk_token=lowerCamelCase__,		**lowerCamelCase__		)

        with open(lowerCamelCase__,		encoding="""utf-8"""		) as vocab_handle:
            A			:	Tuple										=       json.load(lowerCamelCase__		)
        A			:	int										=       {v: k for k, v in self.encoder.items()}
        with open(lowerCamelCase__,		encoding="""utf-8"""		) as merges_handle:
            A			:	Optional[Any]										=       merges_handle.read().split("""\n"""		)[1:-1]
        A			:	Any										=       [tuple(merge.split()		) for merge in merges]
        A			:	str										=       dict(zip(lowerCamelCase__,		range(len(lowerCamelCase__		)		)		)		)
        A			:	int										=       {}






    @property
    def       _lowerCAmelCase (	self		):
        return len(self.encoder		)






    def       _lowerCAmelCase (	self		):
        return dict(self.encoder,		**self.added_tokens_encoder		)






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        if token in self.cache:
            return self.cache[token]
        A			:	Any										=       tuple(lowerCamelCase__		)
        A			:	Optional[int]										=       tuple(list(word[:-1]		) + [word[-1] + """</w>"""]		)
        A			:	Optional[Any]										=       get_pairs(lowerCamelCase__		)

        if not pairs:
            return token

        while True:
            A			:	List[Any]										=       min(lowerCamelCase__,		key=lambda lowerCamelCase__		: self.bpe_ranks.get(lowerCamelCase__,		float("""inf"""		)		)		)
            if bigram not in self.bpe_ranks:
                break
            A					,							A			:	Dict										=       bigram
            A			:	Optional[Any]										=       []
            A			:	List[str]										=       0
            while i < len(lowerCamelCase__		):
                try:
                    A			:	str										=       word.index(lowerCamelCase__,		lowerCamelCase__		)
                except ValueError:
                    new_word.extend(word[i:]		)
                    break
                else:
                    new_word.extend(word[i:j]		)
                    A			:	str										=       j

                if word[i] == first and i < len(lowerCamelCase__		) - 1 and word[i + 1] == second:
                    new_word.append(first + second		)
                    i += 2
                else:
                    new_word.append(word[i]		)
                    i += 1
            A			:	str										=       tuple(lowerCamelCase__		)
            A			:	Union[str, Any]										=       new_word
            if len(lowerCamelCase__		) == 1:
                break
            else:
                A			:	Optional[Any]										=       get_pairs(lowerCamelCase__		)
        A			:	Any										=       """@@ """.join(lowerCamelCase__		)
        A			:	Optional[int]										=       word[:-4]
        A			:	List[Any]										=       word
        return word






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        A			:	Any										=       []

        A			:	str										=       re.findall(R"""\S+\n?""",		lowerCamelCase__		)

        for token in words:
            split_tokens.extend(list(self.bpe(lowerCamelCase__		).split(""" """		)		)		)
        return split_tokens






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        return self.encoder.get(lowerCamelCase__,		self.encoder.get(self.unk_token		)		)






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        return self.decoder.get(lowerCamelCase__,		self.unk_token		)






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        A			:	List[Any]										=       """ """.join(lowerCamelCase__		).replace("""@@ """,		""""""		).strip()
        return out_string






    def       _lowerCAmelCase (	self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        if not os.path.isdir(lowerCamelCase__		):
            logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
            return
        A			:	List[str]										=       os.path.join(
            lowerCamelCase__,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
        A			:	List[str]										=       os.path.join(
            lowerCamelCase__,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

        with open(lowerCamelCase__,		"""w""",		encoding="""utf-8"""		) as f:
            f.write(json.dumps(self.encoder,		indent=2,		sort_keys=lowerCamelCase__,		ensure_ascii=lowerCamelCase__		) + """\n"""		)

        A			:	Any										=       0
        with open(lowerCamelCase__,		"""w""",		encoding="""utf-8"""		) as writer:
            writer.write("""#version: 0.2\n"""		)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),		key=lambda lowerCamelCase__		: kv[1]		):
                if index != token_index:
                    logger.warning(
                        f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                        """ Please check that the tokenizer is not corrupted!"""		)
                    A			:	Union[str, Any]										=       token_index
                writer.write(""" """.join(lowerCamelCase__		) + """\n"""		)
                index += 1

        return vocab_file, merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:Any					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Any					=						{
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			List[str] =				"xlm"
    __lowerCamelCase      :			Tuple =				{
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }






    def __init__(	self,		lowerCamelCase__=3_0145,		lowerCamelCase__=2048,		lowerCamelCase__=12,		lowerCamelCase__=16,		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=True,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=1,		lowerCamelCase__=True,		lowerCamelCase__=512,		lowerCamelCase__=2048**-0.5,		lowerCamelCase__=1e-12,		lowerCamelCase__=0.02,		lowerCamelCase__=0,		lowerCamelCase__=1,		lowerCamelCase__=2,		lowerCamelCase__=3,		lowerCamelCase__=5,		lowerCamelCase__=True,		lowerCamelCase__="first",		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__=0.1,		lowerCamelCase__=5,		lowerCamelCase__=5,		lowerCamelCase__=0,		lowerCamelCase__=0,		lowerCamelCase__=2,		lowerCamelCase__=0,		**lowerCamelCase__,		):
        A			:	Dict										=       vocab_size
        A			:	int										=       emb_dim
        A			:	str										=       n_layers
        A			:	Union[str, Any]										=       n_heads
        A			:	Optional[int]										=       dropout
        A			:	Union[str, Any]										=       attention_dropout
        A			:	Optional[Any]										=       gelu_activation
        A			:	Dict										=       sinusoidal_embeddings
        A			:	int										=       causal
        A			:	Optional[Any]										=       asm
        A			:	Any										=       n_langs
        A			:	List[str]										=       use_lang_emb
        A			:	Union[str, Any]										=       layer_norm_eps
        A			:	str										=       bos_index
        A			:	int										=       eos_index
        A			:	Tuple										=       pad_index
        A			:	str										=       unk_index
        A			:	Optional[Any]										=       mask_index
        A			:	Union[str, Any]										=       is_encoder
        A			:	Tuple										=       max_position_embeddings
        A			:	List[str]										=       embed_init_std
        A			:	Tuple										=       init_std
        A			:	Tuple										=       summary_type
        A			:	int										=       summary_use_proj
        A			:	List[Any]										=       summary_activation
        A			:	Optional[Any]										=       summary_proj_to_labels
        A			:	Optional[Any]										=       summary_first_dropout
        A			:	Optional[int]										=       start_n_top
        A			:	Optional[Any]										=       end_n_top
        A			:	List[str]										=       mask_token_id
        A			:	Tuple										=       lang_id

        if "n_words" in kwargs:
            A			:	List[str]										=       kwargs["""n_words"""]

        super().__init__(pad_token_id=lowerCamelCase__,		bos_token_id=lowerCamelCase__,		**lowerCamelCase__		)






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    @property
    def       _lowerCAmelCase (	self		):
        if self.task == "multiple-choice":
            A			:	Any										=       {0: """batch""", 1: """choice""", 2: """sequence"""}
        else:
            A			:	Union[str, Any]										=       {0: """batch""", 1: """sequence"""}
        return OrderedDict(
            [
                ("""input_ids""", dynamic_axis),
                ("""attention_mask""", dynamic_axis),
                ("""token_type_ids""", dynamic_axis),
            ]		)


