



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCAmelCase: List[Any]							=						logging.get_logger(__name__)

lowerCAmelCase: Dict							=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase: Optional[int]							=						[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

lowerCAmelCase: Dict							=						{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
lowerCAmelCase: Dict							=						{F"funnel-transformer/{name}": 5_1_2 for name in _model_names}
lowerCAmelCase: List[str]							=						{F"funnel-transformer/{name}": {'do_lower_case': True} for name in _model_names}

class 							a__( lowerCamelCase__     ):
    lowercase__			=   VOCAB_FILES_NAMES
    lowercase__			=   PRETRAINED_VOCAB_FILES_MAP
    lowercase__			=   PRETRAINED_INIT_CONFIGURATION
    lowercase__			=   FunnelTokenizer
    lowercase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__			=   2







    def __init__(				self			:  Optional[int]						,      __snake_case			:  Tuple=None						,      __snake_case			:  str=None						,      __snake_case			:  Union[str, Any]=True						,      __snake_case			:  str="<unk>"						,      __snake_case			:  Any="<sep>"						,      __snake_case			:  int="<pad>"						,      __snake_case			:  Union[str, Any]="<cls>"						,      __snake_case			:  Union[str, Any]="<mask>"						,      __snake_case			:  int="<s>"						,      __snake_case			:  Any="</s>"						,      __snake_case			:  Optional[int]=True						,      __snake_case			:  str=True						,      __snake_case			:  List[str]=None						,      __snake_case			:  str="##"						,      **__snake_case			:  List[Any]						,      ):
      super().__init__(
          __snake_case						,      tokenizer_file=__snake_case						,      do_lower_case=__snake_case						,      unk_token=__snake_case						,      sep_token=__snake_case						,      pad_token=__snake_case						,      cls_token=__snake_case						,      mask_token=__snake_case						,      bos_token=__snake_case						,      eos_token=__snake_case						,      clean_text=__snake_case						,      tokenize_chinese_chars=__snake_case						,      strip_accents=__snake_case						,      wordpieces_prefix=__snake_case						,      **__snake_case						,      )

      a						: Tuple					       =					json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
      if (
          normalizer_state.get('lowercase'						,      __snake_case     ) != do_lower_case
          or normalizer_state.get('strip_accents'						,      __snake_case     ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'						,      __snake_case     ) != tokenize_chinese_chars
      ):
        a						: Tuple					       =					getattr(__snake_case						,      normalizer_state.pop('type'     )     )
        a						: Dict					       =					do_lower_case
        a						: Any					       =					strip_accents
        a						: str					       =					tokenize_chinese_chars
        a						: str					       =					normalizer_class(**__snake_case     )

      a						: Optional[Any]					       =					do_lower_case







    def       lowercase_	(				self			:  str						,      __snake_case			:  Dict						,      __snake_case			:  Optional[int]=None     ):
      a						: str					       =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output







    def       lowercase_	(				self			:  List[Any]						,      __snake_case			:  List[int]						,      __snake_case			:  Optional[List[int]] = None     ):
      a						: Any					       =					[self.sep_token_id]
      a						: List[str]					       =					[self.cls_token_id]
      if token_ids_a is None:
        return len(cls     ) * [self.cls_token_type_id] + len(token_ids_a + sep     ) * [0]
      return len(cls     ) * [self.cls_token_type_id] + len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







    def       lowercase_	(				self			:  Union[str, Any]						,      __snake_case			:  str						,      __snake_case			:  Optional[str] = None     ):
      a						: List[str]					       =					self._tokenizer.model.save(__snake_case						,      name=__snake_case     )
      return tuple(__snake_case     )




'''simple docstring'''






import argparse
import os
import re

import packaging.version


lowerCAmelCase: List[str]							=						'examples/'
lowerCAmelCase: List[Any]							=						{
    'examples': (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'),
    'doc': (re.compile(r'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'),
}
lowerCAmelCase: str							=						{
    'init': 'src/transformers/__init__.py',
    'setup': 'setup.py',
}
lowerCAmelCase: str							=						'README.md'
def 				lowerCamelCase__	(						_A			,      _A			,      _A    ):
  with open(_A			,      'r'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    a						: Tuple					       =					f.read()
  a       ,     a						: Tuple					       =					REPLACE_PATTERNS[pattern]
  a						: Dict					       =					replace.replace('VERSION'			,      _A    )
  a						: Dict					       =					re_pattern.sub(_A			,      _A    )
  with open(_A			,      'w'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    f.write(_A    )
def 				lowerCamelCase__	(						_A    ):
  for folder, directories, fnames in os.walk(_A    ):
    # Removing some of the folders with non-actively maintained examples from the walk
    if "research_projects" in directories:
      directories.remove('research_projects'    )
    if "legacy" in directories:
      directories.remove('legacy'    )
    for fname in fnames:
      if fname.endswith('.py'    ):
        update_version_in_file(os.path.join(_A			,      _A    )			,      _A			,      pattern='examples'    )
def 				lowerCamelCase__	(						_A			,      _A=False    ):
  for pattern, fname in REPLACE_FILES.items():
    update_version_in_file(_A			,      _A			,      _A    )
  if not patch:
    update_version_in_examples(_A    )
def 				lowerCamelCase__	(						):
  a						: Tuple					       =					'🤗 Transformers currently provides the following architectures'
  a						: Any					       =					'1. Want to contribute a new model?'
  with open(_A			,      'r'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    a						: Tuple					       =					f.readlines()

  # Find the start of the list.
  a						: Optional[int]					       =					0
  while not lines[start_index].startswith(_start_prompt    ):
    start_index += 1
  start_index += 1

  a						: Optional[int]					       =					start_index
  # Update the lines in the model list.
  while not lines[index].startswith(_end_prompt    ):
    if lines[index].startswith('1.'    ):
      a						: List[Any]					       =					lines[index].replace(
          'https://huggingface.co/docs/transformers/main/model_doc'			,      'https://huggingface.co/docs/transformers/model_doc'			,      )
    index += 1

  with open(_A			,      'w'			,      encoding='utf-8'			,      newline='\n'    ) as f:
    f.writelines(_A    )
def 				lowerCamelCase__	(						):
  with open(REPLACE_FILES['init']			,      'r'    ) as f:
    a						: Union[str, Any]					       =					f.read()
  a						: Tuple					       =					REPLACE_PATTERNS['init'][0].search(_A    ).groups()[0]
  return packaging.version.parse(_A    )
def 				lowerCamelCase__	(						_A=False    ):
  a						: int					       =					get_version()
  if patch and default_version.is_devrelease:
    raise ValueError('Can\'t create a patch version from the dev branch, checkout a released version!'    )
  if default_version.is_devrelease:
    a						: Any					       =					default_version.base_version
  elif patch:
    a						: Dict					       =					f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
  else:
    a						: Union[str, Any]					       =					f"""{default_version.major}.{default_version.minor + 1}.0"""

  # Now let's ask nicely if that's the right one.
  a						: List[Any]					       =					input(f"""Which version are you releasing? [{default_version}]"""    )
  if len(_A    ) == 0:
    a						: Union[str, Any]					       =					default_version

  print(f"""Updating version to {version}."""    )
  global_version_update(_A			,      patch=_A    )
  if not patch:
    print('Cleaning main README, don\'t forget to run `make fix-copies`.'    )
    clean_main_ref_in_model_list()
def 				lowerCamelCase__	(						):
  a						: int					       =					get_version()
  a						: Any					       =					f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
  a						: int					       =					current_version.base_version

  # Check with the user we got that right.
  a						: Tuple					       =					input(f"""Which version are we developing now? [{dev_version}]"""    )
  if len(_A    ) == 0:
    a						: Optional[int]					       =					dev_version

  print(f"""Updating version to {version}."""    )
  global_version_update(_A    )
  print('Cleaning main README, don\'t forget to run `make fix-copies`.'    )
  clean_main_ref_in_model_list()


if __name__ == "__main__":
     lowerCAmelCase: Tuple							=						argparse.ArgumentParser()
     parser.add_argument('--post_release', action='store_true', help='Whether this is pre or post release.')
     parser.add_argument('--patch', action='store_true', help='Whether or not this is a patch release.')
     lowerCAmelCase: Optional[Any]							=						parser.parse_args()
     if not args.post_release:
          pre_release_work(patch=args.patch)
     elif args.patch:
          print('Nothing to do after a patch :-)')
     else:
          post_release_work()