
import importlib
import json
import os
from collections import OrderedDict
from typing import Dict, Optional, Union

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...utils import CONFIG_NAME, FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


_UpperCAmelCase :			int              =	logging.get_logger(__name__)

_UpperCAmelCase :			int              =	OrderedDict(
    [
        ("audio-spectrogram-transformer", "ASTFeatureExtractor"),
        ("beit", "BeitFeatureExtractor"),
        ("chinese_clip", "ChineseCLIPFeatureExtractor"),
        ("clap", "ClapFeatureExtractor"),
        ("clip", "CLIPFeatureExtractor"),
        ("clipseg", "ViTFeatureExtractor"),
        ("conditional_detr", "ConditionalDetrFeatureExtractor"),
        ("convnext", "ConvNextFeatureExtractor"),
        ("cvt", "ConvNextFeatureExtractor"),
        ("data2vec-audio", "Wav2Vec2FeatureExtractor"),
        ("data2vec-vision", "BeitFeatureExtractor"),
        ("deformable_detr", "DeformableDetrFeatureExtractor"),
        ("deit", "DeiTFeatureExtractor"),
        ("detr", "DetrFeatureExtractor"),
        ("dinat", "ViTFeatureExtractor"),
        ("donut-swin", "DonutFeatureExtractor"),
        ("dpt", "DPTFeatureExtractor"),
        ("encodec", "EncodecFeatureExtractor"),
        ("flava", "FlavaFeatureExtractor"),
        ("glpn", "GLPNFeatureExtractor"),
        ("groupvit", "CLIPFeatureExtractor"),
        ("hubert", "Wav2Vec2FeatureExtractor"),
        ("imagegpt", "ImageGPTFeatureExtractor"),
        ("layoutlmv2", "LayoutLMv2FeatureExtractor"),
        ("layoutlmv3", "LayoutLMv3FeatureExtractor"),
        ("levit", "LevitFeatureExtractor"),
        ("maskformer", "MaskFormerFeatureExtractor"),
        ("mctct", "MCTCTFeatureExtractor"),
        ("mobilenet_v1", "MobileNetV1FeatureExtractor"),
        ("mobilenet_v2", "MobileNetV2FeatureExtractor"),
        ("mobilevit", "MobileViTFeatureExtractor"),
        ("nat", "ViTFeatureExtractor"),
        ("owlvit", "OwlViTFeatureExtractor"),
        ("perceiver", "PerceiverFeatureExtractor"),
        ("poolformer", "PoolFormerFeatureExtractor"),
        ("regnet", "ConvNextFeatureExtractor"),
        ("resnet", "ConvNextFeatureExtractor"),
        ("segformer", "SegformerFeatureExtractor"),
        ("sew", "Wav2Vec2FeatureExtractor"),
        ("sew-d", "Wav2Vec2FeatureExtractor"),
        ("speech_to_text", "Speech2TextFeatureExtractor"),
        ("speecht5", "SpeechT5FeatureExtractor"),
        ("swiftformer", "ViTFeatureExtractor"),
        ("swin", "ViTFeatureExtractor"),
        ("swinv2", "ViTFeatureExtractor"),
        ("table-transformer", "DetrFeatureExtractor"),
        ("timesformer", "VideoMAEFeatureExtractor"),
        ("tvlt", "TvltFeatureExtractor"),
        ("unispeech", "Wav2Vec2FeatureExtractor"),
        ("unispeech-sat", "Wav2Vec2FeatureExtractor"),
        ("van", "ConvNextFeatureExtractor"),
        ("videomae", "VideoMAEFeatureExtractor"),
        ("vilt", "ViltFeatureExtractor"),
        ("vit", "ViTFeatureExtractor"),
        ("vit_mae", "ViTFeatureExtractor"),
        ("vit_msn", "ViTFeatureExtractor"),
        ("wav2vec2", "Wav2Vec2FeatureExtractor"),
        ("wav2vec2-conformer", "Wav2Vec2FeatureExtractor"),
        ("wavlm", "Wav2Vec2FeatureExtractor"),
        ("whisper", "WhisperFeatureExtractor"),
        ("xclip", "CLIPFeatureExtractor"),
        ("yolos", "YolosFeatureExtractor"),
    ]
)

_UpperCAmelCase :			Dict              =	_LazyAutoMapping(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)


def 				A	( lowercase			)       ->  List[str]:
  '''simple docstring'''

  for module_name, extractors in FEATURE_EXTRACTOR_MAPPING_NAMES.items():
    if class_name in extractors:
      UpperCamelCase          =					model_type_to_module_name(_snake_case			)

      UpperCamelCase          =					importlib.import_module(f'''.{module_name}'''   ,	'transformers.models'			)
      try:
        return getattr(_snake_case   ,	_snake_case			)
      except AttributeError:
        continue

  for _, extractor in FEATURE_EXTRACTOR_MAPPING._extra_content.items():
    if getattr(_snake_case   ,	'__name__'   ,	_snake_case			) == class_name:
      return extractor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
  UpperCamelCase          =					importlib.import_module('transformers'			)
  if hasattr(_snake_case   ,	_snake_case			):
    return getattr(_snake_case   ,	_snake_case			)

  return None


def 				A	( lowercase   ,	lowercase = None   ,	lowercase = False   ,	lowercase = False   ,	lowercase = None   ,	lowercase = None   ,	lowercase = None   ,	lowercase = False   ,	**lowercase   ,	)       ->  Union[str, Any]:
  '''simple docstring'''

  UpperCamelCase          =					get_file_from_repo(
      _snake_case   ,	_snake_case   ,	cache_dir=_snake_case   ,	force_download=_snake_case   ,	resume_download=_snake_case   ,	proxies=_snake_case   ,	use_auth_token=_snake_case   ,	revision=_snake_case   ,	local_files_only=_snake_case   ,	)
  if resolved_config_file is None:
    logger.info(
        'Could not locate the feature extractor configuration file, will try to use the model config instead.'			)
    return {}

  with open(_snake_case   ,	encoding='utf-8'			) as reader:
    return json.load(_snake_case			)


class     lowercase :



  def __init__(			self	)  -> List[Any]:
    """simple docstring"""
    raise EnvironmentError(
        'AutoFeatureExtractor is designed to be instantiated '
        'using the `AutoFeatureExtractor.from_pretrained(pretrained_model_name_or_path)` method.'	)



  @classmethod
  @replace_list_option_in_docstrings(_a	)
  def __UpperCamelCase			(			cls    ,				A_    ,				**A_	)  -> Any:
    """simple docstring"""
    UpperCamelCase          =					kwargs.pop('config'    ,				_a	)
    UpperCamelCase          =					kwargs.pop('trust_remote_code'    ,				_a	)
    UpperCamelCase          =					True

    UpperCamelCase          =					FeatureExtractionMixin.get_feature_extractor_dict(_a    ,				**_a	)
    UpperCamelCase          =					config_dict.get('feature_extractor_type'    ,				_a	)
    UpperCamelCase          =					None
    if "AutoFeatureExtractor" in config_dict.get('auto_map'    ,				{}	):
      UpperCamelCase          =					config_dict["auto_map"]["AutoFeatureExtractor"]

    # If we don't find the feature extractor class in the feature extractor config, let's try the model config.
    if feature_extractor_class is None and feature_extractor_auto_map is None:
      if not isinstance(_a    ,				_a	):
        UpperCamelCase          =					AutoConfig.from_pretrained(_a    ,				**_a	)
      # It could be in `config.feature_extractor_type``
      UpperCamelCase          =					getattr(_a    ,				'feature_extractor_type'    ,				_a	)
      if hasattr(_a    ,				'auto_map'	) and "AutoFeatureExtractor" in config.auto_map:
        UpperCamelCase          =					config.auto_map["AutoFeatureExtractor"]

    if feature_extractor_class is not None:
      UpperCamelCase          =					feature_extractor_class_from_name(_a	)

    UpperCamelCase          =					feature_extractor_auto_map is not None
    UpperCamelCase          =					feature_extractor_class is not None or type(_a	) in FEATURE_EXTRACTOR_MAPPING
    UpperCamelCase          =					resolve_trust_remote_code(
        _a    ,				_a    ,				_a    ,				_a	)

    if has_remote_code and trust_remote_code:
      UpperCamelCase          =					get_class_from_dynamic_module(
          _a    ,				_a    ,				**_a	)
      UpperCamelCase          =					kwargs.pop('code_revision'    ,				_a	)
      if os.path.isdir(_a	):
        feature_extractor_class.register_for_auto_class()
      return feature_extractor_class.from_dict(_a    ,				**_a	)
    elif feature_extractor_class is not None:
      return feature_extractor_class.from_dict(_a    ,				**_a	)
    # Last try: we use the FEATURE_EXTRACTOR_MAPPING.
    elif type(_a	) in FEATURE_EXTRACTOR_MAPPING:
      UpperCamelCase          =					FEATURE_EXTRACTOR_MAPPING[type(_a	)]
      return feature_extractor_class.from_dict(_a    ,				**_a	)

    raise ValueError(
        F'''Unrecognized feature extractor in {pretrained_model_name_or_path}. Should have a '''
        F'''`feature_extractor_type` key in its {FEATURE_EXTRACTOR_NAME} of {CONFIG_NAME}, or one of the following '''
        F'''`model_type` keys in its {CONFIG_NAME}: {', '.join(c for c in FEATURE_EXTRACTOR_MAPPING_NAMES.keys()	)}'''	)



  @staticmethod
  def __UpperCamelCase			(			A_    ,				A_	)  -> Optional[int]:
    """simple docstring"""
    FEATURE_EXTRACTOR_MAPPING.register(_a    ,				_a	)


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase :			Union[str, Any]              =	get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")


@require_sentencepiece
@require_tokenizers
class     lowercase (		_SCREAMING_SNAKE_CASE  ,       unittest.TestCase       ):
		__lowercase	:	Optional[Any]     					=    GPTSwaTokenizer
		__lowercase	:	Optional[Any]     					=    False
		__lowercase	:	Union[str, Any]     					=    True
		__lowercase	:	Tuple     					=    False



		def __UpperCamelCase			(			self	)  -> Optional[int]:
				"""simple docstring"""
				super().setUp()

				# We have a SentencePiece fixture for testing
				UpperCamelCase          =					GPTSwaTokenizer(A_    ,				eos_token='<unk>'    ,				bos_token='<unk>'    ,				pad_token='<unk>'	)

				tokenizer.save_pretrained(self.tmpdirname	)



		def __UpperCamelCase			(			self    ,				A_	)  -> List[str]:
				"""simple docstring"""
				UpperCamelCase          =					'This is a test'
				UpperCamelCase          =					'This is a test'
				return input_text, output_text



		def __UpperCamelCase			(			self	)  -> int:
				"""simple docstring"""
				UpperCamelCase          =					'<s>'
				UpperCamelCase          =					1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_	)    ,				A_	)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_	)    ,				A_	)



		def __UpperCamelCase			(			self	)  -> Dict:
				"""simple docstring"""
				UpperCamelCase          =					list(self.get_tokenizer().get_vocab().keys()	)

				self.assertEqual(vocab_keys[0]    ,				'<unk>'	)
				self.assertEqual(vocab_keys[1]    ,				'<s>'	)
				self.assertEqual(vocab_keys[-1]    ,				'j'	)
				self.assertEqual(len(A_	)    ,				2_000	)



		def __UpperCamelCase			(			self	)  -> Any:
				"""simple docstring"""
				self.assertEqual(self.get_tokenizer().vocab_size    ,				2_000	)



		def __UpperCamelCase			(			self	)  -> int:
				"""simple docstring"""
				UpperCamelCase          =					GPTSwaTokenizer(A_	)

				UpperCamelCase          =					tokenizer.tokenize('This is a test'	)
				self.assertListEqual(A_    ,				['▁This', '▁is', '▁a', '▁t', 'est']	)

				self.assertListEqual(tokenizer.convert_tokens_to_ids(A_	)    ,				[465, 287, 265, 631, 842]	)

				UpperCamelCase          =					tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
				# fmt: off
				self.assertListEqual(
				    A_    ,				['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']    ,				)
				# fmt: on

				UpperCamelCase          =					tokenizer.convert_tokens_to_ids(A_	)
				self.assertListEqual(
				    A_    ,				[262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]    ,				)

				UpperCamelCase          =					tokenizer.convert_ids_to_tokens(A_	)
				# fmt: off
				self.assertListEqual(
				    A_    ,				['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']	)
				# fmt: on



		def __UpperCamelCase			(			self	)  -> Tuple:
				"""simple docstring"""
				UpperCamelCase          =					GPTSwaTokenizer(A_	)
				UpperCamelCase          =					['This is a test', 'I was born in 92000, and this is falsé.']
				UpperCamelCase          =					[
				    [465, 287, 265, 631, 842],
				    [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
				]

				# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
				for text, expected_ids in zip(A_    ,				A_	):
						self.assertListEqual(tokenizer.encode_fast(A_	)    ,				A_	)

				# Test that decode_fast returns the input text
				for text, token_ids in zip(A_    ,				A_	):
						self.assertEqual(tokenizer.decode_fast(A_	)    ,				A_	)



		@slow
		def __UpperCamelCase			(			self	)  -> List[Any]:
				"""simple docstring"""
				UpperCamelCase          =					[
				    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
				    'Hey there, how are you doing this fine day?',
				    'This is a text with a trailing spaces followed by a dot     .',
				    'Häj sväjs lillebrör! =)',
				    'Det är inget fel på Mr. Cool',
				]

				# fmt: off
				UpperCamelCase          =					{'input_ids': [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
				# fmt: on
				self.tokenizer_integration_test_util(
				    expected_encoding=A_    ,				model_name='AI-Sweden/gpt-sw3-126m'    ,				sequences=A_    ,				)
