

"""simple docstring"""





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_a							=							logging.get_logger(__name__)

_a							=							{'vocab_file': 'spiece.model'}

_a							=							{
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

_a							=							{
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
_a							=							0
_a							=							1
_a							=							2
_a							=							3
_a							=							4




class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 __UpperCAmelCase     : int       		=     VOCAB_FILES_NAMES
 __UpperCAmelCase     : List[Any]       		=     PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase     : List[str]       		=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase     : Optional[int]       		=     "left"


 def __init__(      self							:      Any, UpperCAmelCase__							:      Any, UpperCAmelCase__							:      List[str]=False, UpperCAmelCase__							:      Tuple=True, UpperCAmelCase__							:      Optional[int]=False, UpperCAmelCase__							:      str="<s>", UpperCAmelCase__							:      Any="</s>", UpperCAmelCase__							:      Union[str, Any]="<unk>", UpperCAmelCase__							:      str="<sep>", UpperCAmelCase__							:      Any="<pad>", UpperCAmelCase__							:      int="<cls>", UpperCAmelCase__							:      Optional[int]="<mask>", UpperCAmelCase__							:      Optional[int]=["<eop>", "<eod>"], UpperCAmelCase__							:      Optional[Dict[str, Any]] = None, **UpperCAmelCase__							:      List[str], ):
    # Mask token behave like a normal word, i.e. include the space before it
    __lowercase								=						AddedToken(UpperCAmelCase__, lstrip=UpperCAmelCase__, rstrip=UpperCAmelCase__     ) if isinstance(UpperCAmelCase__, UpperCAmelCase__     ) else mask_token

    __lowercase								=						{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=UpperCAmelCase__, remove_space=UpperCAmelCase__, keep_accents=UpperCAmelCase__, bos_token=UpperCAmelCase__, eos_token=UpperCAmelCase__, unk_token=UpperCAmelCase__, sep_token=UpperCAmelCase__, pad_token=UpperCAmelCase__, cls_token=UpperCAmelCase__, mask_token=UpperCAmelCase__, additional_special_tokens=UpperCAmelCase__, sp_model_kwargs=self.sp_model_kwargs, **UpperCAmelCase__, )

    __lowercase								=						3

    __lowercase								=						do_lower_case
    __lowercase								=						remove_space
    __lowercase								=						keep_accents
    __lowercase								=						vocab_file

    __lowercase								=						spm.SentencePieceProcessor(**self.sp_model_kwargs     )
    self.sp_model.Load(UpperCAmelCase__     )


 @property
 def  _lowercase			(      self							:      Union[str, Any]     ):
    return len(self.sp_model     )


 def  _lowercase			(      self							:      int     ):
    __lowercase								=						{self.convert_ids_to_tokens(UpperCAmelCase__     ): i for i in range(self.vocab_size     )}
    vocab.update(self.added_tokens_encoder     )
    return vocab


 def __getstate__(      self							:      List[Any]     ):
    __lowercase								=						self.__dict__.copy()
    __lowercase								=						None
    return state


 def __setstate__(      self							:      Optional[int], UpperCAmelCase__							:      Tuple     ):
    __lowercase								=						d

    # for backward compatibility
    if not hasattr(self, "sp_model_kwargs"     ):
       __lowercase								=						{}

    __lowercase								=						spm.SentencePieceProcessor(**self.sp_model_kwargs     )
    self.sp_model.Load(self.vocab_file     )


 def  _lowercase			(      self							:      Any, UpperCAmelCase__							:      int     ):
    if self.remove_space:
       __lowercase								=						" ".join(inputs.strip().split()     )
    else:
       __lowercase								=						inputs
    __lowercase								=						outputs.replace("``", "\""     ).replace("''", "\""     )

    if not self.keep_accents:
       __lowercase								=						unicodedata.normalize("NFKD", UpperCAmelCase__     )
       __lowercase								=						"".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__     )]     )
    if self.do_lower_case:
       __lowercase								=						outputs.lower()

    return outputs


 def  _lowercase			(      self							:      Union[str, Any], UpperCAmelCase__							:      str     ):
    __lowercase								=						self.preprocess_text(UpperCAmelCase__     )
    __lowercase								=						self.sp_model.encode(UpperCAmelCase__, out_type=UpperCAmelCase__     )
    __lowercase								=						[]
    for piece in pieces:
       if len(UpperCAmelCase__     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
          __lowercase								=						self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__, ""     )     )
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
             if len(cur_pieces[0]     ) == 1:
                __lowercase								=						cur_pieces[1:]
             else:
                __lowercase								=						cur_pieces[0][1:]
          cur_pieces.append(piece[-1]     )
          new_pieces.extend(UpperCAmelCase__     )
       else:
          new_pieces.append(UpperCAmelCase__     )

    return new_pieces


 def  _lowercase			(      self							:      Any, UpperCAmelCase__							:      Optional[int]     ):
    return self.sp_model.PieceToId(UpperCAmelCase__     )


 def  _lowercase			(      self							:      Union[str, Any], UpperCAmelCase__							:      Tuple     ):
    return self.sp_model.IdToPiece(UpperCAmelCase__     )


 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      List[str]     ):
    __lowercase								=						"".join(UpperCAmelCase__     ).replace(UpperCAmelCase__, " "     ).strip()
    return out_string


 def  _lowercase			(      self							:      List[str], UpperCAmelCase__							:      List[int], UpperCAmelCase__							:      bool = False, UpperCAmelCase__							:      bool = None, UpperCAmelCase__							:      bool = True, **UpperCAmelCase__							:      Union[str, Any], ):
    __lowercase								=						kwargs.pop("use_source_tokenizer", UpperCAmelCase__     )

    __lowercase								=						self.convert_ids_to_tokens(UpperCAmelCase__, skip_special_tokens=UpperCAmelCase__     )

    # To avoid mixing byte-level and unicode for byte-level BPT
    # we need to build string separately for added tokens and byte-level tokens
    # cf. https://github.com/huggingface/transformers/issues/1133
    __lowercase								=						[]
    __lowercase								=						[]
    for token in filtered_tokens:
       if skip_special_tokens and token in self.all_special_ids:
          continue
       if token in self.added_tokens_encoder:
          if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase__     )     )
             __lowercase								=						[]
          sub_texts.append(UpperCAmelCase__     )
       else:
          current_sub_text.append(UpperCAmelCase__     )
    if current_sub_text:
       sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase__     )     )

    # Mimic the behavior of the Rust tokenizer:
    # By default, there are no spaces between special tokens
    __lowercase								=						"".join(UpperCAmelCase__     )

    __lowercase								=						(
        clean_up_tokenization_spaces
        if clean_up_tokenization_spaces is not None
        else self.clean_up_tokenization_spaces
    )
    if clean_up_tokenization_spaces:
       __lowercase								=						self.clean_up_tokenization(UpperCAmelCase__     )
       return clean_text
    else:
       return text


 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      List[int], UpperCAmelCase__							:      Optional[List[int]] = None     ):
    __lowercase								=						[self.sep_token_id]
    __lowercase								=						[self.cls_token_id]
    if token_ids_a is None:
       return token_ids_a + sep + cls
    return token_ids_a + sep + token_ids_a + sep + cls


 def  _lowercase			(      self							:      Any, UpperCAmelCase__							:      List[int], UpperCAmelCase__							:      Optional[List[int]] = None, UpperCAmelCase__							:      bool = False     ):

    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=UpperCAmelCase__, token_ids_a=UpperCAmelCase__, already_has_special_tokens=UpperCAmelCase__     )

    if token_ids_a is not None:
       return ([0] * len(UpperCAmelCase__     )) + [1] + ([0] * len(UpperCAmelCase__     )) + [1, 1]
    return ([0] * len(UpperCAmelCase__     )) + [1, 1]


 def  _lowercase			(      self							:      int, UpperCAmelCase__							:      List[int], UpperCAmelCase__							:      Optional[List[int]] = None     ):
    __lowercase								=						[self.sep_token_id]
    __lowercase								=						[2]

    if token_ids_a is None:
       return len(token_ids_a + sep     ) * [0] + cls_segment_id
    return len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1] + cls_segment_id



 def  _lowercase			(      self							:      str, UpperCAmelCase__							:      str, UpperCAmelCase__							:      Optional[str] = None     ):
    if not os.path.isdir(UpperCAmelCase__     ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
       return
    __lowercase								=						os.path.join(
        UpperCAmelCase__, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

    if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCAmelCase__     ) and os.path.isfile(self.vocab_file     ):
       copyfile(self.vocab_file, UpperCAmelCase__     )
    elif not os.path.isfile(self.vocab_file     ):
       with open(UpperCAmelCase__, "wb"     ) as fi:
          __lowercase								=						self.sp_model.serialized_model_proto()
          fi.write(UpperCAmelCase__     )

    return (out_vocab_file,)







"""simple docstring"""





from __future__ import annotations

from typing import Any




class  _lowerCAmelCase  :







 """simple docstring"""







 def __init__(      self							:      Union[str, Any], UpperCAmelCase__							:      int     ):
    __lowercase								=						num_of_nodes
    __lowercase								=						[]
    __lowercase								=						{}


 def  _lowercase			(      self							:      Tuple, UpperCAmelCase__							:      int, UpperCAmelCase__							:      int, UpperCAmelCase__							:      int     ):
    self.m_edges.append([u_node, v_node, weight]     )


 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      int     ):

    if self.m_component[u_node] == u_node:
       return u_node
    return self.find_component(self.m_component[u_node]     )


 def  _lowercase			(      self							:      Dict, UpperCAmelCase__							:      int     ):

    if self.m_component[u_node] != u_node:
       for k in self.m_component:
          __lowercase								=						self.find_component(UpperCAmelCase__     )


 def  _lowercase			(      self							:      str, UpperCAmelCase__							:      list[int], UpperCAmelCase__							:      int, UpperCAmelCase__							:      int     ):

    if component_size[u_node] <= component_size[v_node]:
       __lowercase								=						v_node
       component_size[v_node] += component_size[u_node]
       self.set_component(UpperCAmelCase__     )

    elif component_size[u_node] >= component_size[v_node]:
       __lowercase								=						self.find_component(UpperCAmelCase__     )
       component_size[u_node] += component_size[v_node]
       self.set_component(UpperCAmelCase__     )



 def  _lowercase			(      self							:      Tuple     ):
    __lowercase								=						[]
    __lowercase								=						0

    __lowercase								=						[-1] * self.m_num_of_nodes

    # A list of components (initialized to all of the nodes)
    for node in range(self.m_num_of_nodes     ):
       self.m_component.update({node: node}     )
       component_size.append(1     )

    __lowercase								=						self.m_num_of_nodes

    while num_of_components > 1:
       for edge in self.m_edges:
          __lowercase		,__lowercase		,__lowercase								=						edge

          __lowercase								=						self.m_component[u]
          __lowercase								=						self.m_component[v]

          if u_component != v_component:

             for component in (u_component, v_component):
                if (
                    minimum_weight_edge[component] == -1
                    or minimum_weight_edge[component][2] > w
                ):
                   __lowercase								=						[u, v, w]

       for edge in minimum_weight_edge:
          if isinstance(UpperCAmelCase__, UpperCAmelCase__     ):
             __lowercase		,__lowercase		,__lowercase								=						edge

             __lowercase								=						self.m_component[u]
             __lowercase								=						self.m_component[v]

             if u_component != v_component:
                mst_weight += w
                self.union(UpperCAmelCase__, UpperCAmelCase__, UpperCAmelCase__     )
                print(F"""Added edge [{u} - {v}]\nAdded weight: {w}\n"""     )
                num_of_components -= 1

       __lowercase								=						[-1] * self.m_num_of_nodes
    print(F"""The total weight of the minimal spanning tree is: {mst_weight}"""     )







def   _A  (  )     ->						None:
   '''simple docstring'''




if __name__ == "__main__":
   import doctest

   doctest.testmod()




