


from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		List[Any]              =  ""
    UpperCAmelCase__						:		int              =  "hf-legacy"  # "hf://"" is reserved for hffs






    def __init__(      self ,       A_ = None ,       A_ = None ,       **A_ ,       )  ->	List[str]:
         super().__init__(self ,       **A_   )
         __UpperCamelCase   							=repo_info
         __UpperCamelCase   							=token
         __UpperCamelCase   							=None






    def _a (      self   )  ->	Union[str, Any]:
         if self.dir_cache is None:
              __UpperCamelCase   							={}
              for hf_file in self.repo_info.siblings:
                   # TODO(QL): add sizes
                   __UpperCamelCase   							={
                       'name': hf_file.rfilename,
                       'size': None,
                       'type': 'file',
                   }
                   self.dir_cache.update(
                       {
                           str(A_   ): {'name': str(A_   ), 'size': None, 'type': 'directory'}
                           for d in list(PurePosixPath(hf_file.rfilename   ).parents   )[:-1]
                       }   )






    def _a (      self ,       A_ ,       A_ = "rb" ,       **A_ ,       )  ->	Dict:
         if not isinstance(self.repo_info ,       A_   ):
              raise NotImplementedError(f'Open is only implemented for dataset repositories, but got {self.repo_info}'   )
         __UpperCamelCase   							=hf_hub_url(self.repo_info.id ,       A_ ,       revision=self.repo_info.sha   )
         return fsspec.open(
             A_ ,       mode=A_ ,       headers=get_authentication_headers_for_url(A_ ,       use_auth_token=self.token   ) ,       client_kwargs={'trust_env': True} ,       ).open()






    def _a (      self ,       A_ ,       **A_   )  ->	Optional[Any]:
         self._get_dirs()
         __UpperCamelCase   							=self._strip_protocol(A_   )
         if path in self.dir_cache:
              return self.dir_cache[path]
         else:
              raise FileNotFoundError(A_   )






    def _a (      self ,       A_ ,       A_=False ,       **A_   )  ->	str:
         self._get_dirs()
         __UpperCamelCase   							=PurePosixPath(path.strip('/'   )   )
         __UpperCamelCase   							={}
         for p, f in self.dir_cache.items():
              __UpperCamelCase   							=PurePosixPath(p.strip('/'   )   )
              __UpperCamelCase   							=p.parent
              if root == path:
                   __UpperCamelCase   							=f
         __UpperCamelCase   							=list(paths.values()   )
         if detail:
              return out
         else:
              return sorted(f['name'] for f in out   )









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_A			=						logging.get_logger(__name__)

_A			=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_A			=						{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

_A			=						{
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

_A			=						{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    UpperCAmelCase__						:		Any              =  VOCAB_FILES_NAMES
    UpperCAmelCase__						:		List[str]              =  PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__						:		Optional[int]              =  PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase__						:		List[str]              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__						:		Dict              =  BertTokenizer






    def __init__(      self ,       A_=None ,       A_=None ,       A_=True ,       A_="[UNK]" ,       A_="[SEP]" ,       A_="[PAD]" ,       A_="[CLS]" ,       A_="[MASK]" ,       A_=True ,       A_=None ,       **A_ ,       )  ->	Any:
         super().__init__(
             A_ ,       tokenizer_file=A_ ,       do_lower_case=A_ ,       unk_token=A_ ,       sep_token=A_ ,       pad_token=A_ ,       cls_token=A_ ,       mask_token=A_ ,       tokenize_chinese_chars=A_ ,       strip_accents=A_ ,       **A_ ,       )

         __UpperCamelCase   							=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
         if (
             normalizer_state.get('lowercase' ,       A_   ) != do_lower_case
             or normalizer_state.get('strip_accents' ,       A_   ) != strip_accents
             or normalizer_state.get('handle_chinese_chars' ,       A_   ) != tokenize_chinese_chars
         ):
              __UpperCamelCase   							=getattr(A_ ,       normalizer_state.pop('type'   )   )
              __UpperCamelCase   							=do_lower_case
              __UpperCamelCase   							=strip_accents
              __UpperCamelCase   							=tokenize_chinese_chars
              __UpperCamelCase   							=normalizer_class(**A_   )

         __UpperCamelCase   							=do_lower_case






    def _a (      self ,       A_ ,       A_=None   )  ->	List[str]:
         __UpperCamelCase   							=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

         return output






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]
         if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def _a (      self ,       A_ ,       A_ = None   )  ->	Tuple[str]:
         __UpperCamelCase   							=self._tokenizer.model.save(A_ ,       name=A_   )
         return tuple(A_   )





