


import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


_A			=						logging.getLogger(__name__)

_A			=						'pytorch_model.bin'






@dataclasses.dataclass
class    UpperCAmelCase__	:
    UpperCAmelCase__						:		List[str]              =  dataclasses.field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models."}		)
    UpperCAmelCase__						:		Union[str, Any]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co."}       ,  )






@dataclasses.dataclass
class    UpperCAmelCase__	:
    UpperCAmelCase__						:		Dict              =  dataclasses.field(metadata={"help": "A csv or a json file containing the training data."}		)
    UpperCAmelCase__						:		str              =  dataclasses.field(metadata={"help": "A csv or a json file containing the data to predict on."}		)
    UpperCAmelCase__						:		Union[str, Any]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "A csv or a json file containing the validation data."}		)
    UpperCAmelCase__						:		Tuple              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "The name of the task to train on."}       ,  )
    UpperCAmelCase__						:		Optional[int]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "The list of labels for the task."}		)






@dataclasses.dataclass
class    UpperCAmelCase__	:
    UpperCAmelCase__						:		List[Any]              =  dataclasses.field(
        metadata={"help": "The output directory where the model predictions and checkpoints will be written."}		)
    UpperCAmelCase__						:		Any              =  dataclasses.field(
        default="accuracy"       ,  metadata={"help": "The evaluation metric used for the task."}		)
    UpperCAmelCase__						:		str              =  dataclasses.field(
        default="no"       ,  metadata={
            "help": "The evaluation strategy to adopt during training. Possible values are: [\"no\", \"step\", \"epoch]"
        }       ,  )
    UpperCAmelCase__						:		int              =  dataclasses.field(
        default=1_0       ,  metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}       ,  )
    UpperCAmelCase__						:		List[str]              =  dataclasses.field(
        default=0.0       ,  metadata={
            "help": "How much the specified evaluation metric must improve to satisfy early stopping conditions."
        }       ,  )
    UpperCAmelCase__						:		Tuple              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "Whether to filter the pseudo-labeled data based on the confidence score."}       ,  )
    UpperCAmelCase__						:		Union[str, Any]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "Whether to filter the pseudo-labeled data based on the validation performance."}       ,  )
    UpperCAmelCase__						:		Union[str, Any]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "Whether to fine-tune on labeled data after pseudo training."}       ,  )
    UpperCAmelCase__						:		Union[str, Any]              =  dataclasses.field(
        default=0.0       ,  metadata={"help": "Confidence threshold for pseudo-labeled data filtering."}       ,  )
    UpperCAmelCase__						:		str              =  dataclasses.field(
        default=1_0_0       ,  metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}       ,  )
    UpperCAmelCase__						:		List[Any]              =  dataclasses.field(
        default=__lowercase       ,  metadata={"help": "Random seed for initialization."}       ,  )

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Optional[int]      ,					SCREAMING_SNAKE_CASE__      : Tuple      ,					SCREAMING_SNAKE_CASE__      : Any      ,					SCREAMING_SNAKE_CASE__      : Tuple      ,					SCREAMING_SNAKE_CASE__      : int      ,					SCREAMING_SNAKE_CASE__      : Any    ):
     __UpperCamelCase   							=datasets.concatenate_datasets([infer_input, infer_output]      ,					axis=1    )

     if args.do_filter_by_confidence:
          __UpperCamelCase   							=dataset.filter(lambda SCREAMING_SNAKE_CASE__    : example["probability"] > args.confidence_threshold    )

     if args.do_filter_by_val_performance:
          assert eval_result >= 0.0 and eval_result <= 1.0
          __UpperCamelCase   							=int(eval_result * len(__a    )    )
          print(__a    )
          __UpperCamelCase   							=dataset.sort('probability'      ,					reverse=__a    )
          __UpperCamelCase   							=dataset.select(range(__a    )    )

     __UpperCamelCase   							=dataset.remove_columns(['label', 'probability']    )
     __UpperCamelCase   							=dataset.rename_column('prediction'      ,					'label'    )
     __UpperCamelCase   							=dataset.map(lambda SCREAMING_SNAKE_CASE__    : {"label": idalabel[example["label"]]}    )
     __UpperCamelCase   							=dataset.shuffle(seed=args.seed    )

     __UpperCamelCase   							=os.path.join(__a      ,					F'train_pseudo.{args.data_file_extension}'    )
     if args.data_file_extension == "csv":
          dataset.to_csv(__a      ,					index=__a    )
     else:
          dataset.to_json(__a    )





def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : List[Any]      ,					SCREAMING_SNAKE_CASE__      : str      ,					SCREAMING_SNAKE_CASE__      : Dict      ,					SCREAMING_SNAKE_CASE__      : str      ,					**SCREAMING_SNAKE_CASE__      : Union[str, Any]    ):
     __UpperCamelCase   							=Accelerator()
     # Make one log on every process with the configuration for debugging.
     logging.basicConfig(
         format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'      ,					datefmt='%m/%d/%Y %H:%M:%S'      ,					level=logging.INFO      ,					)
     logger.info(accelerator.state    )

     # Setup logging, we only want one process per machine to log things on the
     # screen. accelerator.is_local_main_process is only True for one process per
     # machine.
     logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR    )

     if accelerator.is_local_main_process:
          datasets.utils.logging.set_verbosity_warning()
          transformers.utils.logging.set_verbosity_info()
     else:
          datasets.utils.logging.set_verbosity_error()
          transformers.utils.logging.set_verbosity_error()

     __UpperCamelCase   							=STModelArguments(model_name_or_path=__a    )
     __UpperCamelCase   							=STDataArguments(train_file=__a      ,					infer_file=__a    )
     __UpperCamelCase   							=STTrainingArguments(output_dir=__a    )
     __UpperCamelCase   							=argparse.Namespace()

     for arg_class in (model_args, data_args, training_args):
          for key, value in vars(__a    ).items():
               setattr(__a      ,					__a      ,					__a    )

     for key, value in kwargs.items():
          if hasattr(__a      ,					__a    ):
               setattr(__a      ,					__a      ,					__a    )

    # Sanity checks
     __UpperCamelCase   							={}
     __UpperCamelCase   							=None

     # You need to provide the training data and the data to predict on
     assert args.train_file is not None
     assert args.infer_file is not None
     __UpperCamelCase   							=args.train_file
     __UpperCamelCase   							=args.infer_file

     if args.evaluation_strategy != IntervalStrategy.NO.value:
          assert args.eval_file is not None
          __UpperCamelCase   							=args.eval_file

     for key in data_files:
          __UpperCamelCase   							=data_files[key].split('.'    )[-1]
          assert extension in ["csv", "json"], F'`{key}_file` should be a csv or a json file.'
          if args.data_file_extension is None:
               __UpperCamelCase   							=extension
          else:
               assert extension == args.data_file_extension, F'`{key}_file` should be a {args.data_file_extension} file`.'

     assert (
         args.eval_metric in datasets.list_metrics()
     ), F'{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}.'

     # If passed along, set the training seed now.
     if args.seed is not None:
          set_seed(args.seed    )

     logger.info('Creating the initial data directory for self-training...'    )
     __UpperCamelCase   							=F'{args.output_dir}/self-train_iter-{{}}'.format
     __UpperCamelCase   							=data_dir_format(0    )

     if accelerator.is_main_process:
          if args.output_dir is not None:
               os.makedirs(args.output_dir      ,					exist_ok=__a    )
               os.makedirs(__a      ,					exist_ok=__a    )
     accelerator.wait_for_everyone()

     __UpperCamelCase   							=None
     __UpperCamelCase   							=None
     __UpperCamelCase   							=0
     __UpperCamelCase   							=False
     # Show the progress bar
     __UpperCamelCase   							=tqdm(range(args.max_selftrain_iterations    )      ,					disable=not accelerator.is_local_main_process    )

     # Self-train
     for iteration in range(0      ,					int(args.max_selftrain_iterations    )    ):
          __UpperCamelCase   							=data_dir_format(__a    )
          assert os.path.exists(__a    )

          # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
          # iteration > 0
          __UpperCamelCase   							=os.path.join(__a      ,					'stage-1'    )
          __UpperCamelCase   							={
              'accelerator': accelerator,
              'model_name_or_path': args.model_name_or_path,
              'cache_dir': args.cache_dir,
              'do_train': True,
              'train_file': data_files['train'] if iteration == 0 else data_files['train_pseudo'],
              'do_eval': True if args.eval_file is not None else False,
              'eval_file': data_files['eval'],
              'do_predict': True,
              'infer_file': data_files['infer'],
              'task_name': args.task_name,
              'label_list': args.label_list,
              'output_dir': current_output_dir,
              'eval_metric': args.eval_metric,
              'evaluation_strategy': args.evaluation_strategy,
              'early_stopping_patience': args.early_stopping_patience,
              'early_stopping_threshold': args.early_stopping_threshold,
              'seed': args.seed,
          }
          # Add additional training arguments
          for key, value in kwargs.items():
               if key not in arguments_dict and not hasattr(__a      ,					__a    ):
                    arguments_dict.update({key: value}    )

          __UpperCamelCase   							=os.path.join(__a      ,					'best-checkpoint'      ,					__a    )
          if os.path.exists(__a    ):
               logger.info(
                   'Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1.'      ,					__a      ,					__a      ,					)
          else:
               logger.info('***** Running self-training: iteration: %d, stage: 1 *****'      ,					__a    )
               finetune(**__a    )
               accelerator.wait_for_everyone()
               assert os.path.exists(__a    )
               logger.info('Self-training job completed: iteration: %d, stage: 1.'      ,					__a    )

          if iteration > 0 and args.finetune_on_labeled_data:
               # Stage 2 (optional): fine-tuning on the original labeled data
               __UpperCamelCase   							=os.path.join(__a      ,					'best-checkpoint'    )
               __UpperCamelCase   							=os.path.join(__a      ,					'stage-2'    )
               # Update arguments_dict
               __UpperCamelCase   							=model_path
               __UpperCamelCase   							=data_files['train']
               __UpperCamelCase   							=current_output_dir

               __UpperCamelCase   							=os.path.join(__a      ,					'best-checkpoint'      ,					__a    )
               if os.path.exists(__a    ):
                    logger.info(
                        'Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2.'      ,					__a      ,					__a      ,					)
               else:
                    logger.info('***** Running self-training: iteration: %d, stage: 2 *****'      ,					__a    )
                    finetune(**__a    )
                    accelerator.wait_for_everyone()
                    assert os.path.exists(__a    )
                    logger.info('Self-training job completed: iteration: %d, stage: 2.'      ,					__a    )

          __UpperCamelCase   							=iteration
          __UpperCamelCase   							=data_dir_format(iteration + 1    )

          __UpperCamelCase   							=AutoConfig.from_pretrained(os.path.join(__a      ,					'best-checkpoint'    )    )
          __UpperCamelCase   							=config.idalabel
          __UpperCamelCase   							=os.path.join(__a      ,					'eval_results_best-checkpoint.json'    )
          __UpperCamelCase   							=os.path.join(__a      ,					'test_results_best-checkpoint.json'    )
          assert os.path.exists(__a    )

          with open(__a      ,					'r'    ) as f:
               __UpperCamelCase   							=float(json.load(__a    )[args.eval_metric]    )
          __UpperCamelCase   							=os.path.join(__a      ,					'infer_output_best-checkpoint.csv'    )
          assert os.path.exists(__a    )
          # Loading the dataset from local csv or json files.
          __UpperCamelCase   							=load_dataset(args.data_file_extension      ,					data_files={'data': data_files['infer']}    )['data']
          __UpperCamelCase   							=load_dataset('csv'      ,					data_files={'data': infer_output_file}    )['data']

          if accelerator.is_main_process:
               os.makedirs(__a      ,					exist_ok=__a    )
               shutil.copy(__a      ,					os.path.join(__a      ,					F'eval_results_iter-{iteration}.json'    )    )
               if os.path.exists(__a    ):
                    shutil.copy(__a      ,					os.path.join(__a      ,					F'test_results_iter-{iteration}.json'    )    )
               create_pseudo_labeled_data(__a      ,					__a      ,					__a      ,					__a      ,					__a      ,					__a    )
          accelerator.wait_for_everyone()

          __UpperCamelCase   							=os.path.join(__a      ,					F'train_pseudo.{args.data_file_extension}'    )

          if args.evaluation_strategy != IntervalStrategy.NO.value:
               __UpperCamelCase   							=eval_result

               if best_iteration is None:
                    __UpperCamelCase   							=new_iteration
                    __UpperCamelCase   							=new_eval_result
               else:
                    if new_eval_result - best_eval_result > args.early_stopping_threshold:
                         __UpperCamelCase   							=new_iteration
                         __UpperCamelCase   							=new_eval_result
                         __UpperCamelCase   							=0
                    else:
                         if new_eval_result == best_eval_result:
                              __UpperCamelCase   							=new_iteration
                              __UpperCamelCase   							=new_eval_result
                         early_stopping_patience_counter += 1

                    if early_stopping_patience_counter >= args.early_stopping_patience:
                         __UpperCamelCase   							=True

          progress_bar.update(1    )

          if should_training_stop:
               break

     if best_iteration is not None:
          # Save the best iteration
          logger.info('Best iteration: %d'      ,					__a    )
          logger.info('Best evaluation result: %s = %f'      ,					args.eval_metric      ,					__a    )
          accelerator.wait_for_everyone()
          if accelerator.is_main_process:
               shutil.copy(
                   os.path.join(__a      ,					F'eval_results_iter-{iteration}.json'    )      ,					os.path.join(__a      ,					'eval_results_best-iteration.json'    )      ,					)
     else:
          # Assume that the last iteration is the best
          logger.info('Best iteration: %d'      ,					args.max_selftrain_iterations - 1    )
          logger.info('Best evaluation result: %s = %f'      ,					args.eval_metric      ,					__a    )
          accelerator.wait_for_everyone()
          if accelerator.is_main_process:
               shutil.copy(
                   os.path.join(__a      ,					F'eval_results_iter-{args.max_selftrain_iterations - 1}.json'    )      ,					os.path.join(__a      ,					'eval_results_best-iteration.json'    )      ,					)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_A			=						{'configuration_vit': ['VIT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ViTConfig', 'ViTOnnxConfig']}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A			=						['ViTFeatureExtractor']
   _A			=						['ViTImageProcessor']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A			=						[
       'VIT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'ViTForImageClassification',
       'ViTForMaskedImageModeling',
       'ViTModel',
       'ViTPreTrainedModel',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A			=						[
       'TFViTForImageClassification',
       'TFViTModel',
       'TFViTPreTrainedModel',
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A			=						[
       'FlaxViTForImageClassification',
       'FlaxViTModel',
       'FlaxViTPreTrainedModel',
   ]

if TYPE_CHECKING:
   from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_vit import ViTFeatureExtractor
      from .image_processing_vit import ViTImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_vit import (
          VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
          ViTForImageClassification,
          ViTForMaskedImageModeling,
          ViTModel,
          ViTPreTrainedModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
   import sys

   _A			=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





