




import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
_lowerCamelCase    :       str        =							"src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
_lowerCamelCase    :       Dict        =							direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
_lowerCamelCase    :       Optional[Any]        =							re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
_lowerCamelCase    :       List[str]        =							re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
_lowerCamelCase    :       Optional[int]        =							re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
_lowerCamelCase    :       int        =							[
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]

def       a__					(		UpperCAmelCase							:      Optional[Any]      )  ->						Dict:
  UpperCAmelCase	:							List[Any]						=			re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''    ,			__a      )
  return [m.group(0      ) for m in matches]

def       a__					(		)  ->						Dict:
  UpperCAmelCase	:							Union[str, Any]						=			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
  UpperCAmelCase	:							Any						=			{
      config.replace('''Config'''    ,			''''''      ): model_type for model_type, config in config_maping_names.items()
  }

  # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
  UpperCAmelCase	:							List[Any]						=			collections.defaultdict(__a      )
  UpperCAmelCase	:							int						=			collections.defaultdict(__a      )
  UpperCAmelCase	:							Union[str, Any]						=			collections.defaultdict(__a      )

  # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
  for attr_name in dir(__a      ):
    UpperCAmelCase	:							Union[str, Any]						=			None
    if _re_tf_models.match(__a      ) is not None:
      UpperCAmelCase	:							Any						=			tf_models
      UpperCAmelCase	:							str						=			_re_tf_models.match(__a      ).groups()[0]
    elif _re_flax_models.match(__a      ) is not None:
      UpperCAmelCase	:							List[str]						=			flax_models
      UpperCAmelCase	:							str						=			_re_flax_models.match(__a      ).groups()[0]
    elif _re_pt_models.match(__a      ) is not None:
      UpperCAmelCase	:							List[str]						=			pt_models
      UpperCAmelCase	:							Dict						=			_re_pt_models.match(__a      ).groups()[0]

    if lookup_dict is not None:
      while len(__a      ) > 0:
        if attr_name in model_prefix_to_model_type:
          UpperCAmelCase	:							Union[str, Any]						=			True
          break
        # Try again after removing the last word in the name
        UpperCAmelCase	:							List[str]						=			''''''.join(camel_case_split(__a      )[:-1]      )

  UpperCAmelCase	:							int						=			set(list(pt_models.keys()      ) + list(tf_models.keys()      ) + list(flax_models.keys()      )      )
  UpperCAmelCase	:							Optional[int]						=			list(__a      )
  all_models.sort()

  UpperCAmelCase	:							Any						=			{'''model_type''': all_models}
  UpperCAmelCase	:							Optional[Any]						=			[pt_models[t] for t in all_models]
  UpperCAmelCase	:							List[str]						=			[tf_models[t] for t in all_models]
  UpperCAmelCase	:							Optional[Any]						=			[flax_models[t] for t in all_models]

  # Now let's use the auto-mapping names to make sure
  UpperCAmelCase	:							Dict						=			{}
  for t in all_models:
    if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
      UpperCAmelCase	:							List[str]						=			'''AutoProcessor'''
    elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
      UpperCAmelCase	:							Optional[Any]						=			'''AutoTokenizer'''
    elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
      UpperCAmelCase	:							List[str]						=			'''AutoFeatureExtractor'''
    else:
      # Default to AutoTokenizer if a model has nothing, for backward compatibility.
      UpperCAmelCase	:							str						=			'''AutoTokenizer'''

  UpperCAmelCase	:							Union[str, Any]						=			[processors[t] for t in all_models]

  return pd.DataFrame(__a      )

def       a__					(		UpperCAmelCase							:      str      )  ->						str:
  UpperCAmelCase	:							Any						=			[
      transformers_module.models.auto.modeling_auto,
      transformers_module.models.auto.modeling_tf_auto,
      transformers_module.models.auto.modeling_flax_auto,
  ]
  for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
    UpperCAmelCase	:							List[Any]						=			[model_mapping, f'''TF_{model_mapping}''', f'''FLAX_{model_mapping}''']
    UpperCAmelCase	:							int						=			[auto_class, f'''TF_{auto_class}''', f'''Flax_{auto_class}''']
    # Loop through all three frameworks
    for module, cls, mapping in zip(__a    ,			__a    ,			__a      ):
      # The type of pipeline may not exist in this framework
      if not hasattr(__a    ,			__a      ):
        continue
      # First extract all model_names
      UpperCAmelCase	:							Union[str, Any]						=			[]
      for name in getattr(__a    ,			__a      ).values():
        if isinstance(__a    ,			__a      ):
          model_names.append(__a      )
        else:
          model_names.extend(list(__a      )      )

            # Add pipeline tag and auto model class for those models
      table.update({model_name: (pipeline_tag, cls) for model_name in model_names}      )

  return table

def       a__					(		UpperCAmelCase							:      Union[str, Any]    ,			UpperCAmelCase							:      List[str]      )  ->						Optional[Any]:
  UpperCAmelCase	:							int						=			get_frameworks_table()
  UpperCAmelCase	:							Optional[int]						=			Dataset.from_pandas(__a      )

  UpperCAmelCase	:							str						=			hf_hub_download(
      '''huggingface/transformers-metadata'''    ,			'''pipeline_tags.json'''    ,			repo_type='''dataset'''    ,			token=__a      )
  UpperCAmelCase	:							Tuple						=			Dataset.from_json(__a      )
  UpperCAmelCase	:							Optional[int]						=			{
      tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
      for i in range(len(__a      )      )
  }
  UpperCAmelCase	:							Union[str, Any]						=			update_pipeline_and_auto_class_table(__a      )

  # Sort the model classes to avoid some nondeterministic updates to create false update commits.
  UpperCAmelCase	:							Dict						=			sorted(table.keys()      )
  UpperCAmelCase	:							str						=			pd.DataFrame(
      {
          '''model_class''': model_classes,
          '''pipeline_tag''': [table[m][0] for m in model_classes],
          '''auto_class''': [table[m][1] for m in model_classes],
      }      )
  UpperCAmelCase	:							Any						=			Dataset.from_pandas(__a      )

  with tempfile.TemporaryDirectory() as tmp_dir:
    frameworks_dataset.to_json(os.path.join(__a    ,			'''frameworks.json'''      )      )
    tags_dataset.to_json(os.path.join(__a    ,			'''pipeline_tags.json'''      )      )

    if commit_sha is not None:
      UpperCAmelCase	:							Dict						=			(
          f'''Update with commit {commit_sha}\n\nSee: '''
          f'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
      )
    else:
      UpperCAmelCase	:							int						=			'''Update'''

    upload_folder(
        repo_id='''huggingface/transformers-metadata'''    ,			folder_path=__a    ,			repo_type='''dataset'''    ,			token=__a    ,			commit_message=__a    ,			)

def       a__					(		)  ->						int:
  UpperCAmelCase	:							Any						=			{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
  UpperCAmelCase	:							Union[str, Any]						=			transformers_module.pipelines.SUPPORTED_TASKS
  UpperCAmelCase	:							Optional[Any]						=			[]
  for key in pipeline_tasks:
    if key not in in_table:
      UpperCAmelCase	:							Optional[Any]						=			pipeline_tasks[key]['''pt''']
      if isinstance(__a    ,			(list, tuple)      ):
        UpperCAmelCase	:							Union[str, Any]						=			model[0]
      UpperCAmelCase	:							Any						=			model.__name__
      if model not in in_table.values():
        missing.append(__a      )

  if len(__a      ) > 0:
    UpperCAmelCase	:							Optional[int]						=			''', '''.join(__a      )
    raise ValueError(
        '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
        f'''`utils/update_metadata.py`: {msg}. Please add them!'''      )


if __name__ == "__main__":
     _lowerCamelCase    :       Optional[int]        =							argparse.ArgumentParser()
     parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
     parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
     parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
     _lowerCamelCase    :       str        =							parser.parse_args()

     if args.check_only:
          check_pipeline_tags()
     else:
          update_metadata(args.token, args.commit_sha)








import logging
import os

from .state import PartialState





class  __UpperCAmelCase					(	logging.LoggerAdapter						):



   @staticmethod
   def 	__magic_name__   (							__A							:					str						):
     UpperCAmelCase	:							Dict						=			PartialState()
     return not main_process_only or (main_process_only and state.is_main_process)





   def 	__magic_name__   (							self							:					Union[str, Any],		__A							:					Union[str, Any],		__A							:					Union[str, Any],		*__A							:					Optional[int],		**__A							:					Tuple						):
     if PartialState._shared_state == {}:
       raise RuntimeError(
           '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''						)
     UpperCAmelCase	:							List[str]						=			kwargs.pop('''main_process_only''',		__A						)
     UpperCAmelCase	:							int						=			kwargs.pop('''in_order''',		__A						)

     if self.isEnabledFor(__A						):
       if self._should_log(__A						):
         UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			self.process(__A,		__A						)
         self.logger.log(__A,		__A,		*__A,		**__A						)

       elif in_order:
         UpperCAmelCase	:							Dict						=			PartialState()
         for i in range(state.num_processes						):
           if i == state.process_index:
             UpperCAmelCase			,						UpperCAmelCase	:							Optional[int]						=			self.process(__A,		__A						)
             self.logger.log(__A,		__A,		*__A,		**__A						)
           state.wait_for_everyone()






def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      str = None      )  ->						Dict:
  if log_level is None:
    UpperCAmelCase	:							Union[str, Any]						=			os.environ.get('''ACCELERATE_LOG_LEVEL'''    ,			UpperCAmelCase      )
  UpperCAmelCase	:							Tuple						=			logging.getLogger(UpperCAmelCase      )
  if log_level is not None:
    logger.setLevel(log_level.upper()      )
    logger.root.setLevel(log_level.upper()      )
  return MultiProcessAdapter(UpperCAmelCase    ,			{}      )





