


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_albert import AlbertTokenizer
else:
   _A			=						None

_A			=						logging.get_logger(__name__)
_A			=						{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

_A			=						{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

_A			=						{
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

_A			=						'▁'






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		List[str]              =  VOCAB_FILES_NAMES
    UpperCAmelCase__						:		Optional[Any]              =  PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__						:		Any              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__						:		Tuple              =  AlbertTokenizer






    def __init__(      self ,       A_=None ,       A_=None ,       A_=True ,       A_=True ,       A_=False ,       A_="[CLS]" ,       A_="[SEP]" ,       A_="<unk>" ,       A_="[SEP]" ,       A_="<pad>" ,       A_="[CLS]" ,       A_="[MASK]" ,       **A_ ,       )  ->	Optional[Any]:
         # Mask token behave like a normal word, i.e. include the space before it and
         # is included in the raw text, there should be a match in a non-normalized sentence.
         __UpperCamelCase   							=(
             AddedToken(A_ ,       lstrip=A_ ,       rstrip=A_ ,       normalized=A_   )
             if isinstance(A_ ,       A_   )
             else mask_token
         )

         super().__init__(
             A_ ,       tokenizer_file=A_ ,       do_lower_case=A_ ,       remove_space=A_ ,       keep_accents=A_ ,       bos_token=A_ ,       eos_token=A_ ,       unk_token=A_ ,       sep_token=A_ ,       pad_token=A_ ,       cls_token=A_ ,       mask_token=A_ ,       **A_ ,       )

         __UpperCamelCase   							=do_lower_case
         __UpperCamelCase   							=remove_space
         __UpperCamelCase   							=keep_accents
         __UpperCamelCase   							=vocab_file
         __UpperCamelCase   							=False if not self.vocab_file else True






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]
         if token_ids_a is None:
              return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def _a (      self ,       A_ ,       A_ = None   )  ->	Tuple[str]:
         if not self.can_save_slow_tokenizer:
              raise ValueError(
                  'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                  'tokenizer.'   )

         if not os.path.isdir(A_   ):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
              return
         __UpperCamelCase   							=os.path.join(
             A_ ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(A_   ):
              copyfile(self.vocab_file ,       A_   )

         return (out_vocab_file,)









import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Features    ):
     __UpperCamelCase   							=np.inf

     def set_batch_size(SCREAMING_SNAKE_CASE__      : FeatureType    ) -> None:
          nonlocal batch_size
          if isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ):
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS    )
          elif isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ):
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS    )
          elif isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    ) and feature.dtype == "binary":
               __UpperCamelCase   							=min(SCREAMING_SNAKE_CASE__      ,					config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS    )

     _visit(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__    )

     return None if batch_size is np.inf else batch_size






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_ = None ,       A_ = None ,       A_ = None ,       A_ = False ,       A_ = False ,       A_ = None ,       **A_ ,       )  ->	Dict:
         super().__init__(
             A_ ,       split=A_ ,       features=A_ ,       cache_dir=A_ ,       keep_in_memory=A_ ,       streaming=A_ ,       num_proc=A_ ,       **A_ ,       )
         __UpperCamelCase   							=path_or_paths if isinstance(A_ ,       A_   ) else {self.split: path_or_paths}
         __UpperCamelCase   							=_PACKAGED_DATASETS_MODULES['parquet'][1]
         __UpperCamelCase   							=Parquet(
             cache_dir=A_ ,       data_files=A_ ,       features=A_ ,       hash=A_ ,       **A_ ,       )






    def _a (      self   )  ->	List[Any]:
         # Build iterable dataset
         if self.streaming:
              __UpperCamelCase   							=self.builder.as_streaming_dataset(split=self.split   )
         # Build regular (map-style) dataset
         else:
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None
              __UpperCamelCase   							=None

              self.builder.download_and_prepare(
                  download_config=A_ ,       download_mode=A_ ,       verification_mode=A_ ,       base_path=A_ ,       num_proc=self.num_proc ,       )
              __UpperCamelCase   							=self.builder.as_dataset(
                  split=self.split ,       verification_mode=A_ ,       in_memory=self.keep_in_memory   )
         return dataset






class    UpperCAmelCase__	:



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_ ,       A_ = None ,       **A_ ,       )  ->	List[Any]:
         __UpperCamelCase   							=dataset
         __UpperCamelCase   							=path_or_buf
         __UpperCamelCase   							=batch_size or get_writer_batch_size(dataset.features   )
         __UpperCamelCase   							=parquet_writer_kwargs






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

         if isinstance(self.path_or_buf ,       (str, bytes, os.PathLike)   ):
              with open(self.path_or_buf ,       'wb+'   ) as buffer:
                   __UpperCamelCase   							=self._write(file_obj=A_ ,       batch_size=A_ ,       **self.parquet_writer_kwargs   )
         else:
              __UpperCamelCase   							=self._write(file_obj=self.path_or_buf ,       batch_size=A_ ,       **self.parquet_writer_kwargs   )
         return written






    def _a (      self ,       A_ ,       A_ ,       **A_   )  ->	int:
         __UpperCamelCase   							=0
         __UpperCamelCase   							=parquet_writer_kwargs.pop('path_or_buf' ,       A_   )
         __UpperCamelCase   							=self.dataset.features.arrow_schema

         __UpperCamelCase   							=pq.ParquetWriter(A_ ,       schema=A_ ,       **A_   )

         for offset in logging.tqdm(
             range(0 ,       len(self.dataset   ) ,       A_   ) ,       unit='ba' ,       disable=not logging.is_progress_bar_enabled() ,       desc='Creating parquet from Arrow format' ,       ):
              __UpperCamelCase   							=query_table(
                  table=self.dataset._data ,       key=slice(A_ ,       offset + batch_size   ) ,       indices=self.dataset._indices if self.dataset._indices is not None else None ,       )
              writer.write_table(A_   )
              written += batch.nbytes
         writer.close()
         return written





