






import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCamelCase			    =     '''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase			    =     direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCamelCase			    =     transformers.models.auto.configuration_auto.CONFIG_MAPPING

UpperCamelCase			    =     {
    # used to compute the property `self.chunk_length`
    '''EncodecConfig''': ['''overlap'''],
    # used as `self.bert_model = BertModel(config, ...)`
    '''DPRConfig''': True,
    # not used in modeling files, but it's an important information
    '''FSMTConfig''': ['''langs'''],
    # used internally in the configuration class file
    '''GPTNeoConfig''': ['''attention_types'''],
    # used internally in the configuration class file
    '''EsmConfig''': ['''is_folding_model'''],
    # used during training (despite we don't have training script for these models yet)
    '''Mask2FormerConfig''': ['''ignore_value'''],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    '''OneFormerConfig''': ['''ignore_value''', '''norm'''],
    # used during preprocessing and collation, see `collating_graphormer.py`
    '''GraphormerConfig''': ['''spatial_pos_max'''],
    # used internally in the configuration class file
    '''T5Config''': ['''feed_forward_proj'''],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    '''MT5Config''': ['''feed_forward_proj''', '''tokenizer_class'''],
    '''UMT5Config''': ['''feed_forward_proj''', '''tokenizer_class'''],
    # used internally in the configuration class file
    '''LongT5Config''': ['''feed_forward_proj'''],
    # used internally in the configuration class file
    '''SwitchTransformersConfig''': ['''feed_forward_proj'''],
    # having default values other than `1e-5` - we can't fix them without breaking
    '''BioGptConfig''': ['''layer_norm_eps'''],
    # having default values other than `1e-5` - we can't fix them without breaking
    '''GLPNConfig''': ['''layer_norm_eps'''],
    # having default values other than `1e-5` - we can't fix them without breaking
    '''SegformerConfig''': ['''layer_norm_eps'''],
    # having default values other than `1e-5` - we can't fix them without breaking
    '''CvtConfig''': ['''layer_norm_eps'''],
    # having default values other than `1e-5` - we can't fix them without breaking
    '''PerceiverConfig''': ['''layer_norm_eps'''],
    # used internally to calculate the feature size
    '''InformerConfig''': ['''num_static_real_features''', '''num_time_features'''],
    # used internally to calculate the feature size
    '''TimeSeriesTransformerConfig''': ['''num_static_real_features''', '''num_time_features'''],
    # used internally to calculate the feature size
    '''AutoformerConfig''': ['''num_static_real_features''', '''num_time_features'''],
    # used internally to calculate `mlp_dim`
    '''SamVisionConfig''': ['''mlp_ratio'''],
    # For (head) training, but so far not implemented
    '''ClapAudioConfig''': ['''num_classes'''],
    # Not used, but providing useful information to users
    '''SpeechT5HifiGanConfig''': ['''sampling_rate'''],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        '''CLIPSegConfig''': True,
        '''DeformableDetrConfig''': True,
        '''DetaConfig''': True,
        '''DinatConfig''': True,
        '''DonutSwinConfig''': True,
        '''EfficientFormerConfig''': True,
        '''FSMTConfig''': True,
        '''JukeboxConfig''': True,
        '''LayoutLMv2Config''': True,
        '''MaskFormerSwinConfig''': True,
        '''MT5Config''': True,
        '''NatConfig''': True,
        '''OneFormerConfig''': True,
        '''PerceiverConfig''': True,
        '''RagConfig''': True,
        '''SpeechT5Config''': True,
        '''SwinConfig''': True,
        '''Swin2SRConfig''': True,
        '''Swinv2Config''': True,
        '''SwitchTransformersConfig''': True,
        '''TableTransformerConfig''': True,
        '''TapasConfig''': True,
        '''TransfoXLConfig''': True,
        '''UniSpeechConfig''': True,
        '''UniSpeechSatConfig''': True,
        '''WavLMConfig''': True,
        '''WhisperConfig''': True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        '''JukeboxPriorConfig''': True,
        # TODO: @Younes (for `is_decoder`)
        '''Pix2StructTextConfig''': True,
    }
)

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__	,snake_case__		)							->  Any:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  False
						for attribute in attributes:
												for modeling_source in source_strings:
																		# check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
																		if (
																		    F'config.{attribute}' in modeling_source
																		    or F'getattr(config, "{attribute}"' in modeling_source
																		    or F'getattr(self.config, "{attribute}"' in modeling_source
																		):
																								_SCREAMING_SNAKE_CASE     				=  True
																		# Deal with multi-line cases
																		elif (
																		    re.search(
																		        rF'getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*"{attribute}"'	,__lowerCAmelCase	,)
																		    is not None
																		):
																								_SCREAMING_SNAKE_CASE     				=  True
																		# `SequenceSummary` is called with `SequenceSummary(config)`
																		elif attribute in [
																		    "summary_type",
																		    "summary_use_proj",
																		    "summary_activation",
																		    "summary_last_dropout",
																		    "summary_proj_to_labels",
																		    "summary_first_dropout",
																		]:
																								if "SequenceSummary" in modeling_source:
																														_SCREAMING_SNAKE_CASE     				=  True
																		if attribute_used:
																								break
												if attribute_used:
																		break

    # common and important attributes, even if they do not always appear in the modeling files
						_SCREAMING_SNAKE_CASE     				=  [
						    '''bos_index''',
						    '''eos_index''',
						    '''pad_index''',
						    '''unk_index''',
						    '''mask_index''',
						    '''image_size''',
						    '''use_cache''',
						    '''out_features''',
						    '''out_indices''',
						]
						_SCREAMING_SNAKE_CASE     				=  ['''encoder_no_repeat_ngram_size''']

						# Special cases to be allowed
						_SCREAMING_SNAKE_CASE     				=  True
						if not attribute_used:
												_SCREAMING_SNAKE_CASE     				=  False
												for attribute in attributes:
																		# Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
																		if attribute in ["is_encoder_decoder"] and default_value is True:
																								_SCREAMING_SNAKE_CASE     				=  True
																		elif attribute in ["tie_word_embeddings"] and default_value is False:
																								_SCREAMING_SNAKE_CASE     				=  True

																		# Allow cases without checking the default value in the configuration class
																		elif attribute in attributes_to_allow + attributes_used_in_generation:
																								_SCREAMING_SNAKE_CASE     				=  True
																		elif attribute.endswith("""_token_id"""		):
																								_SCREAMING_SNAKE_CASE     				=  True

																		# configuration class specific cases
																		if not case_allowed:
																								_SCREAMING_SNAKE_CASE     				=  SPECIAL_CASES_TO_ALLOW.get(config_class.__name__	,[]		)
																								_SCREAMING_SNAKE_CASE     				=  allowed_cases is True or attribute in allowed_cases

						return attribute_used or case_allowed

def 	__lowerCamelCase      (		snake_case__		)							->  str:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  dict(inspect.signature(config_class.__init__		).parameters		)
						_SCREAMING_SNAKE_CASE     				=  [x for x in list(signature.keys()		) if x not in ['''self''', '''kwargs''']]
						_SCREAMING_SNAKE_CASE     				=  [signature[param].default for param in parameter_names]

						# If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
						# as one variant is used, the test should pass
						_SCREAMING_SNAKE_CASE     				=  {}
						if len(config_class.attribute_map		) > 0:
												_SCREAMING_SNAKE_CASE     				=  {v: k for k, v in config_class.attribute_map.items()}

						# Get the path to modeling source files
						_SCREAMING_SNAKE_CASE     				=  inspect.getsourcefile(__lowerCAmelCase		)
						_SCREAMING_SNAKE_CASE     				=  os.path.dirname(__lowerCAmelCase		)
						# Let's check against all frameworks: as long as one framework uses an attribute, we are good.
						_SCREAMING_SNAKE_CASE     				=  [os.path.join(__lowerCAmelCase	,__lowerCAmelCase		) for fn in os.listdir(__lowerCAmelCase		) if fn.startswith("""modeling_"""		)]

						# Get the source code strings
						_SCREAMING_SNAKE_CASE     				=  []
						for path in modeling_paths:
												if os.path.isfile(__lowerCAmelCase		):
																		with open(__lowerCAmelCase		) as fp:
																								modeling_sources.append(fp.read()		)

						_SCREAMING_SNAKE_CASE     				=  []
						for config_param, default_value in zip(__lowerCAmelCase	,__lowerCAmelCase		):
												# `attributes` here is all the variant names for `config_param`
												_SCREAMING_SNAKE_CASE     				=  [config_param]
												# some configuration classes have non-empty `attribute_map`, and both names could be used in the
												# corresponding modeling files. As long as one of them appears, it is fine.
												if config_param in reversed_attribute_map:
																		attributes.append(reversed_attribute_map[config_param]		)

												if not check_attribute_being_used(__lowerCAmelCase	,__lowerCAmelCase	,__lowerCAmelCase	,__lowerCAmelCase		):
																		unused_attributes.append(attributes[0]		)

						return sorted(__lowerCAmelCase		)

def 	__lowerCamelCase      (		)							->  Optional[Any]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  {}
						for _config_class in list(CONFIG_MAPPING.values()		):
												# Skip deprecated models
												if "models.deprecated" in _config_class.__module__:
																		continue
												# Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
												_SCREAMING_SNAKE_CASE     				=  [
												    cls
												    for name, cls in inspect.getmembers(
												        inspect.getmodule(_config_class		)	,lambda snake_case__		: inspect.isclass(__lowerCAmelCase		)
												        and issubclass(__lowerCAmelCase	,__lowerCAmelCase		)
												        and inspect.getmodule(__lowerCAmelCase		) == inspect.getmodule(_config_class		)	,)
												]
												for config_class in config_classes_in_module:
																		_SCREAMING_SNAKE_CASE     				=  check_config_attributes_being_used(__lowerCAmelCase		)
																		if len(__lowerCAmelCase		) > 0:
																								_SCREAMING_SNAKE_CASE     				=  unused_attributes

						if len(__lowerCAmelCase		) > 0:
												_SCREAMING_SNAKE_CASE     				=  '''The following configuration classes contain unused attributes in the corresponding modeling files:\n'''
												for name, attributes in configs_with_unused_attributes.items():
																		error += F'{name}: {attributes}\n'

												raise ValueError(__lowerCAmelCase		)


if __name__ == "__main__":
	check_config_attributes()









# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


UpperCamelCase			    =     get_logger(__name__)



class 				__UpperCAmelCase		:
			__snake_case    :				Tuple									=							"dummy_data"
			__snake_case    :				List[Any]									=							"datasets"
			__snake_case    :				List[Any]									=							False







			def __init__(     self:  Any				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Union[Version, str]				,     UpperCAmelCase_:  Optional[str] = None				,     UpperCAmelCase_:  bool = False				,     UpperCAmelCase_:  bool = True				,     UpperCAmelCase_:  Optional[List[Callable]] = None				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  0
									_SCREAMING_SNAKE_CASE     				=  dataset_name
									_SCREAMING_SNAKE_CASE     				=  cache_dir
									_SCREAMING_SNAKE_CASE     				=  use_local_dummy_data
									_SCREAMING_SNAKE_CASE     				=  config
									# download_callbacks take a single url as input
									_SCREAMING_SNAKE_CASE     				=  download_callbacks or []
									# if False, it doesn't load existing files and it returns the paths of the dummy files relative
									# to the dummy_data zip file root
									_SCREAMING_SNAKE_CASE     				=  load_existing_dummy_data

									# TODO(PVP, QL) might need to make this more general
									_SCREAMING_SNAKE_CASE     				=  str(UpperCAmelCase_					)
									# to be downloaded
									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  None







			@property
			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									if self._dummy_file is None:
															_SCREAMING_SNAKE_CASE     				=  self.download_dummy_data()
									return self._dummy_file







			@property
			def 							UpperCamelCase				(     self:  Union[str, Any]					):







									'''simple docstring'''




									if self.config is not None:
															# structure is dummy / config_name / version_name
															return os.path.join("""dummy"""				,     self.config.name				,     self.version_name					)
									# structure is dummy / version_name
									return os.path.join("""dummy"""				,     self.version_name					)







			@property
			def 							UpperCamelCase				(     self:  Optional[int]					):







									'''simple docstring'''




									return os.path.join(self.dummy_data_folder				,     """dummy_data.zip"""					)







			def 							UpperCamelCase				(     self:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  (
									    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
									)

									_SCREAMING_SNAKE_CASE     				=  cached_path(
									    UpperCAmelCase_				,     cache_dir=self.cache_dir				,     extract_compressed_file=UpperCAmelCase_				,     force_extract=UpperCAmelCase_					)

									return os.path.join(UpperCAmelCase_				,     self.dummy_file_name					)







			@property
			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									return os.path.join(self.datasets_scripts_dir				,     self.dataset_name				,     self.dummy_zip_file					)







			@property
			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									if self._bucket_url is None:
															_SCREAMING_SNAKE_CASE     				=  hf_github_url(self.dataset_name				,     self.dummy_zip_file.replace(os.sep				,     """/"""					)					)
									return self._bucket_url







			@property
			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									if os.path.isdir(self.dummy_file					):
															return self.dummy_file
									# else cut off path to file -> example `xsum`.
									return "/".join(self.dummy_file.replace(os.sep				,     """/"""					).split("""/"""					)[:-1]					)







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  str				,     *UpperCAmelCase_:  Dict					):







									'''simple docstring'''




									if self.load_existing_dummy_data:
															# dummy data is downloaded and tested
															_SCREAMING_SNAKE_CASE     				=  self.dummy_file
									else:
															# dummy data cannot be downloaded and only the path to dummy file is returned
															_SCREAMING_SNAKE_CASE     				=  self.dummy_file_name

									# special case when data_url is a dict
									if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
															return self.create_dummy_data_dict(UpperCAmelCase_				,     UpperCAmelCase_					)
									elif isinstance(UpperCAmelCase_				,     (list, tuple)					):
															return self.create_dummy_data_list(UpperCAmelCase_				,     UpperCAmelCase_					)
									else:
															return self.create_dummy_data_single(UpperCAmelCase_				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  Tuple				,     *UpperCAmelCase_:  Any					):







									'''simple docstring'''




									return self.download_and_extract(UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									return self.download_and_extract(UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  Dict				,     UpperCAmelCase_:  Dict				,     *UpperCAmelCase_:  Tuple				,     **UpperCAmelCase_:  Union[str, Any]					):







									'''simple docstring'''




									return path







			def 							UpperCamelCase				(     self:  str					):







									'''simple docstring'''




									return {}







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {}
									for key, single_urls in data_url.items():
															for download_callback in self.download_callbacks:
																					if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
																											for single_url in single_urls:
																																	download_callback(UpperCAmelCase_					)
																					else:
																											_SCREAMING_SNAKE_CASE     				=  single_urls
																											download_callback(UpperCAmelCase_					)
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
															if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  [os.path.join(UpperCAmelCase_				,     urllib.parse.quote_plus(Path(UpperCAmelCase_					).name					)					) for x in single_urls]
															else:
																					_SCREAMING_SNAKE_CASE     				=  single_urls
																					_SCREAMING_SNAKE_CASE     				=  os.path.join(UpperCAmelCase_				,     urllib.parse.quote_plus(Path(UpperCAmelCase_					).name					)					)
															_SCREAMING_SNAKE_CASE     				=  value

									# make sure that values are unique
									if all(isinstance(UpperCAmelCase_				,     UpperCAmelCase_					) for i in dummy_data_dict.values()					) and len(set(dummy_data_dict.values()					)					) < len(
									    dummy_data_dict.values()					):
															# append key to value to make its name unique
															_SCREAMING_SNAKE_CASE     				=  {key: value + key for key, value in dummy_data_dict.items()}

									return dummy_data_dict







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  []
									# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
									_SCREAMING_SNAKE_CASE     				=  all(bool(re.findall("""[0-9]{3,}-of-[0-9]{3,}"""				,     UpperCAmelCase_					)					) for url in data_url					)
									_SCREAMING_SNAKE_CASE     				=  all(
									    url.startswith("""https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"""					) for url in data_url					)
									if data_url and (is_tf_records or is_pubmed_records):
															_SCREAMING_SNAKE_CASE     				=  [data_url[0]] * len(UpperCAmelCase_					)
									for single_url in data_url:
															for download_callback in self.download_callbacks:
																					download_callback(UpperCAmelCase_					)
															# we force the name of each key to be the last file / folder name of the url path
															# if the url has arguments, we need to encode them with urllib.parse.quote_plus
															_SCREAMING_SNAKE_CASE     				=  os.path.join(UpperCAmelCase_				,     urllib.parse.quote_plus(single_url.split("""/"""					)[-1]					)					)
															dummy_data_list.append(UpperCAmelCase_					)
									return dummy_data_list







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									for download_callback in self.download_callbacks:
															download_callback(UpperCAmelCase_					)
									# we force the name of each key to be the last file / folder name of the url path
									# if the url has arguments, we need to encode them with urllib.parse.quote_plus
									_SCREAMING_SNAKE_CASE     				=  os.path.join(UpperCAmelCase_				,     urllib.parse.quote_plus(data_url.split("""/"""					)[-1]					)					)
									if os.path.exists(UpperCAmelCase_					) or not self.load_existing_dummy_data:
															return value
									else:
															# Backward compatibility, maybe deprecate at one point.
															# For many datasets with single url calls to dl_manager.download_and_extract,
															# the dummy_data.zip file is actually the zipped downloaded file
															# while now we expected the dummy_data.zip file to be a directory containing
															# the downloaded file.
															return path_to_dummy_data







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									pass







			def 							UpperCamelCase				(     self:  Optional[Any]					):







									'''simple docstring'''




									pass







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  Union[str, Any]					):







									'''simple docstring'''




									def _iter_archive_members(UpperCAmelCase_:  Any					):
															# this preserves the order of the members inside the ZIP archive
															_SCREAMING_SNAKE_CASE     				=  Path(self.dummy_file					).parent
															_SCREAMING_SNAKE_CASE     				=  path.relative_to(UpperCAmelCase_					)
															with ZipFile(self.local_path_to_dummy_data					) as zip_file:
																					_SCREAMING_SNAKE_CASE     				=  zip_file.namelist()
															for member in members:
																					if member.startswith(relative_path.as_posix()					):
																											yield dummy_parent_path.joinpath(UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  Path(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  _iter_archive_members(UpperCAmelCase_					) if self.use_local_dummy_data else path.rglob("""*"""					)
									for file_path in file_paths:
															if file_path.is_file() and not file_path.name.startswith((""".""", """__""")					):
																					yield file_path.relative_to(UpperCAmelCase_					).as_posix(), file_path.open("""rb"""					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									if not isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
															_SCREAMING_SNAKE_CASE     				=  [paths]
									for path in paths:
															if os.path.isfile(UpperCAmelCase_					):
																					if os.path.basename(UpperCAmelCase_					).startswith((""".""", """__""")					):
																											return
																					yield path
															else:
																					for dirpath, dirnames, filenames in os.walk(UpperCAmelCase_					):
																											if os.path.basename(UpperCAmelCase_					).startswith((""".""", """__""")					):
																																	continue
																											dirnames.sort()
																											for filename in sorted(UpperCAmelCase_					):
																																	if filename.startswith((""".""", """__""")					):
																																							continue
																																	yield os.path.join(UpperCAmelCase_				,     UpperCAmelCase_					)

