




'''simple docstring'''



import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowerCamelCase_			=						os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(f'''{bindir}/../../examples/pytorch/translation'''):
 from run_translation import main  # noqa


set_seed(42)
lowerCamelCase_			=						'sshleifer/student_marian_en_ro_6_1'
lowerCamelCase_			=						'sshleifer/tiny-mbart'







@require_torch
class 		lowercase_   (			A       ):


   """simple docstring"""

   def 							lowerCAmelCase_      (   self :     Dict  ,				__lowerCamelCase :     List[Any]=False  ,				__lowerCamelCase :     List[str]=None  ,				__lowerCamelCase :     str=True  ,				__lowerCamelCase :     Optional[Any]=True  ,				__lowerCamelCase :     Any=True  ,				__lowerCamelCase :     List[str]=True  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.run_trainer(
          eval_steps=1  ,				max_len=1_2  ,				model_name=__lowerCamelCase  ,				num_train_epochs=1  ,				distributed=__lowerCamelCase  ,				extra_args_str=__lowerCamelCase  ,				predict_with_generate=__lowerCamelCase  ,				do_train=__lowerCamelCase  ,				do_eval=__lowerCamelCase  ,				do_predict=__lowerCamelCase  ,				)
      _SCREAMING_SNAKE_CASE	=				TrainerState.load_from_json(os.path.join(__lowerCamelCase  ,				"trainer_state.json"   )   ).log_history

      if not do_eval:
         return

      _SCREAMING_SNAKE_CASE	=				[log for log in logs if "eval_loss" in log.keys()]

      _SCREAMING_SNAKE_CASE	=				eval_metrics[0]
      if predict_with_generate:
         assert "eval_bleu" in first_step_stats

         _SCREAMING_SNAKE_CASE	=				eval_metrics[-1]
         assert isinstance(last_step_stats["eval_bleu"]  ,				__lowerCamelCase   )
         assert not math.isnan(float(last_step_stats["eval_loss"]   )   ), "eval_loss must not be `nan`"



   @require_torch_non_multi_gpu
   def 							lowerCAmelCase_      (   self :     str   ):

      """simple docstring"""
      self.run_seqaseq_quick()



   @require_torch_multi_gpu
   def 							lowerCAmelCase_      (   self :     Union[str, Any]   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase   )



   @require_torch_multi_gpu
   def 							lowerCAmelCase_      (   self :     Optional[int]   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase   )



   @unittest.skip("Requires an update of the env running those tests"   )
   @require_torch_multi_gpu
   @require_fairscale
   def 							lowerCAmelCase_      (   self :     str   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase  ,				extra_args_str="--sharded_ddp simple"   )



   @unittest.skip("Requires an update of the env running those tests"   )
   @require_torch_multi_gpu
   @require_fairscale
   def 							lowerCAmelCase_      (   self :     str   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase  ,				extra_args_str="--sharded_ddp simple --fp16"   )



   @unittest.skip("Requires an update of the env running those tests"   )
   @require_torch_multi_gpu
   @require_fairscale
   def 							lowerCAmelCase_      (   self :     str   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase  ,				extra_args_str="--sharded_ddp zero_dp_2"  ,				predict_with_generate=__lowerCamelCase   )



   @unittest.skip("Requires an update of the env running those tests"   )
   @require_torch_multi_gpu
   @require_fairscale
   def 							lowerCAmelCase_      (   self :     List[Any]   ):

      """simple docstring"""
      self.run_seqaseq_quick(
          distributed=__lowerCamelCase  ,				extra_args_str="--sharded_ddp zero_dp_2 --fp16"  ,				predict_with_generate=__lowerCamelCase   )



   @require_apex
   @require_torch_gpu
   def 							lowerCAmelCase_      (   self :     Any   ):

      """simple docstring"""
      self.run_seqaseq_quick(distributed=__lowerCamelCase  ,				extra_args_str="--fp16 --fp16_backend=apex"   )
      # test 2nd time - was getting eval_loss': nan'
      # to reproduce the problem set distributed=False
      self.run_seqaseq_quick(distributed=__lowerCamelCase  ,				extra_args_str="--fp16 --fp16_backend=apex"   )



   @parameterized.expand(["base", "low", "high", "mixed"]   )
   @require_torch_multi_gpu
   def 							lowerCAmelCase_      (   self :     List[Any]  ,				__lowerCamelCase :     Optional[Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				{
          # test with the default log_level - should be info and thus log info once
          "base": {"extra_args_str": "", "n_matches": 1},
          # test with low log_level and log_level_replica - should be noisy on all processes
          # now the info string should appear twice on 2 processes
          "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
          # test with high log_level and low log_level_replica
          # now the info string should appear once only on the replica
          "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
          # test with high log_level and log_level_replica - should be quiet on all processes
          "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
      }

      _SCREAMING_SNAKE_CASE	=				experiments[experiment_id]
      _SCREAMING_SNAKE_CASE	=				{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
      _SCREAMING_SNAKE_CASE	=				"Running training"
      with CaptureStderr() as cl:
         self.run_seqaseq_quick(**__lowerCamelCase  ,				extra_args_str=data["extra_args_str"]   )
      _SCREAMING_SNAKE_CASE	=				len(re.findall(__lowerCamelCase  ,				cl.err   )   )
      self.assertEqual(__lowerCamelCase  ,				data["n_matches"]   )



   @slow
   def 							lowerCAmelCase_      (   self :     List[Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.run_trainer(
          eval_steps=2  ,				max_len=1_2_8  ,				model_name=__lowerCamelCase  ,				learning_rate=3e-4  ,				num_train_epochs=1_0  ,				distributed=__lowerCamelCase  ,				)

      # Check metrics
      _SCREAMING_SNAKE_CASE	=				TrainerState.load_from_json(os.path.join(__lowerCamelCase  ,				"trainer_state.json"   )   ).log_history
      _SCREAMING_SNAKE_CASE	=				[log for log in logs if "eval_loss" in log.keys()]
      _SCREAMING_SNAKE_CASE	=				eval_metrics[0]
      _SCREAMING_SNAKE_CASE	=				eval_metrics[-1]

      assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
      assert isinstance(last_step_stats["eval_bleu"]  ,				__lowerCamelCase   )

      # test if do_predict saves generations and metrics
      _SCREAMING_SNAKE_CASE	=				os.listdir(__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				{os.path.basename(__lowerCamelCase   ) for p in contents}
      assert "generated_predictions.txt" in contents
      assert "predict_results.json" in contents



   @slow
   @require_bitsandbytes
   def 							lowerCAmelCase_      (   self :     List[Any]   ):

      """simple docstring"""
      from transformers.training_args import OptimizerNames

      def train_and_return_metrics(__lowerCamelCase :     str   ) -> Tuple[int, float]:
         _SCREAMING_SNAKE_CASE	=				"--skip_memory_metrics 0"

         _SCREAMING_SNAKE_CASE	=				self.run_trainer(
             max_len=1_2_8  ,				model_name=__lowerCamelCase  ,				learning_rate=3e-4  ,				num_train_epochs=1  ,				optim=__lowerCamelCase  ,				distributed=__lowerCamelCase  ,				extra_args_str=__lowerCamelCase  ,				do_eval=__lowerCamelCase  ,				do_predict=__lowerCamelCase  ,				n_gpus_to_use=1  ,				)

         # Check metrics
         _SCREAMING_SNAKE_CASE	=				TrainerState.load_from_json(Path(__lowerCamelCase  ,				"trainer_state.json"   )   ).log_history
         _SCREAMING_SNAKE_CASE	=				int(logs[0]["train_mem_gpu_peaked_delta"] / 2**2_0   )
         _SCREAMING_SNAKE_CASE	=				int(logs[0]["train_mem_gpu_alloc_delta"] / 2**2_0   )

         _SCREAMING_SNAKE_CASE	=				logs[0]["train_loss"]
         return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

      _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value   )
      _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value   )

      _SCREAMING_SNAKE_CASE	=				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

      _SCREAMING_SNAKE_CASE	=				gpu_peak_mem_orig + gpu_alloc_mem_orig
      _SCREAMING_SNAKE_CASE	=				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
      _SCREAMING_SNAKE_CASE	=				gpu_total_mem_orig - gpu_total_mem_bnb

      # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
      # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
      # in 2 bytes and the diff in optim memory usage is derived as so:
      #
      # - normal 25*8=~200MB (8 bytes per param)
      # - bnb    25*2= ~50MB (2 bytes per param)
      #
      # Thus we should expect ~150MB total memory saved.
      #
      # Peak memory should be the same - the total should be different by about that same margin
      #
      # After leaving a small margin to accommodate for differences between gpus let's check
      # that we have at least 120MB in savings
      _SCREAMING_SNAKE_CASE	=				1_2_0

      # uncomment the following if this test starts failing - requires py38 for a new print feature
      # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
      # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
      # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
      # print(f"{gpu_alloc_mem_diff=}MB")
      # print(f"{gpu_peak_mem_diff=}MB")
      # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
      # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

      self.assertGreater(
          __lowerCamelCase  ,				__lowerCamelCase  ,				"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
          F""" a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and"""
          F""" gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB"""  ,				)

      self.assertGreater(
          __lowerCamelCase  ,				__lowerCamelCase  ,				"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
          F""" a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and"""
          F""" gpu_total_mem_bnb={gpu_total_mem_bnb}MB"""  ,				)

      self.assertEqual(
          __lowerCamelCase  ,				__lowerCamelCase  ,				F"""loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}"""   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     int  ,				__lowerCamelCase :     float = 3e-3  ,				__lowerCamelCase :     str = "adafactor"  ,				__lowerCamelCase :     bool = False  ,				__lowerCamelCase :     str = None  ,				__lowerCamelCase :     int = 0  ,				__lowerCamelCase :     bool = True  ,				__lowerCamelCase :     bool = True  ,				__lowerCamelCase :     bool = True  ,				__lowerCamelCase :     bool = True  ,				__lowerCamelCase :     int = None  ,				):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
      _SCREAMING_SNAKE_CASE	=				self.get_auto_remove_tmp_dir()
      _SCREAMING_SNAKE_CASE	=				F"""
            --model_name_or_path {model_name}
            --train_file {data_dir}/train.json
            --validation_file {data_dir}/val.json
            --test_file {data_dir}/test.json
            --output_dir {output_dir}
            --overwrite_output_dir
            --max_train_samples 8
            --max_source_length {max_len}
            --max_target_length {max_len}
            --do_train
            --num_train_epochs {str(__lowerCamelCase   )}
            --per_device_train_batch_size 4
            --learning_rate {learning_rate}
            --warmup_steps 8
            --logging_steps 0
            --logging_strategy no
            --save_steps {str(__lowerCamelCase   )}
            --group_by_length
            --label_smoothing_factor 0.1
            --target_lang ro_RO
            --source_lang en_XX
        """.split()

      _SCREAMING_SNAKE_CASE	=				F"""
            --do_eval
            --per_device_eval_batch_size 4
            --max_eval_samples 8
            --val_max_target_length {max_len}
            --evaluation_strategy steps
            --eval_steps {str(__lowerCamelCase   )}
        """.split()

      _SCREAMING_SNAKE_CASE	=				"\n            --do_predict\n        ".split()

      _SCREAMING_SNAKE_CASE	=				[]
      if do_train:
         args += args_train

      if do_eval:
         args += args_eval

      if do_predict:
         args += args_predict

      if predict_with_generate:
         args += "--predict_with_generate".split()

      if do_train:
         if optim == "adafactor":
            args += "--adafactor".split()
         else:
            args += F"""--optim {optim}""".split()

      if extra_args_str is not None:
         args += extra_args_str.split()

      if distributed:
         if n_gpus_to_use is None:
            _SCREAMING_SNAKE_CASE	=				get_gpu_count()
         _SCREAMING_SNAKE_CASE	=				get_torch_dist_unique_port()
         _SCREAMING_SNAKE_CASE	=				F"""
                -m torch.distributed.run
                --nproc_per_node={n_gpus_to_use}
                --master_port={master_port}
                {self.examples_dir_str}/pytorch/translation/run_translation.py
            """.split()
         _SCREAMING_SNAKE_CASE	=				[sys.executable] + distributed_args + args
         # keep for quick debug
         # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
         execute_subprocess_async(__lowerCamelCase  ,				env=self.get_env()   )
      else:
         _SCREAMING_SNAKE_CASE	=				["run_translation.py"] + args
         with patch.object(__lowerCamelCase  ,				"argv"  ,				__lowerCamelCase   ):
            main()

      return output_dir







'''simple docstring'''



import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


lowerCamelCase_			=						logging.get_logger(__name__)







class 		lowercase_   (			A       ):


   """simple docstring"""

   lowerCamelCase_      =						'''linear'''
   lowerCamelCase_      =						'''cosine'''
   lowerCamelCase_      =						'''cosine_with_restarts'''
   lowerCamelCase_      =						'''polynomial'''
   lowerCamelCase_      =						'''constant'''
   lowerCamelCase_      =						'''constant_with_warmup'''
   lowerCamelCase_      =						'''piecewise_constant'''


def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optimizer	,					__A    :			int = -1  )			->			Optional[int]:
   return LambdaLR(__A	,					lambda __A  : 1	,					last_epoch=__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optimizer	,					__A    :			int	,					__A    :			int = -1  )			->			Dict:

   def lr_lambda(__A    :			int  ):
      if current_step < num_warmup_steps:
         return float(__A  ) / float(max(1.0	,					__A  )  )
      return 1.0

   return LambdaLR(__A	,					__A	,					last_epoch=__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optimizer	,					__A    :			str	,					__A    :			int = -1  )			->			Tuple:
   _SCREAMING_SNAKE_CASE	=				{}
   _SCREAMING_SNAKE_CASE	=				step_rules.split(","  )
   for rule_str in rule_list[:-1]:
      _SCREAMING_SNAKE_CASE,	_SCREAMING_SNAKE_CASE	=				rule_str.split(":"  )
      _SCREAMING_SNAKE_CASE	=				int(__A  )
      _SCREAMING_SNAKE_CASE	=				float(__A  )
      _SCREAMING_SNAKE_CASE	=				value
   _SCREAMING_SNAKE_CASE	=				float(rule_list[-1]  )

   def create_rules_function(__A    :			Tuple	,					__A    :			List[Any]  ):
      def rule_func(__A    :			int  ) -> float:
         _SCREAMING_SNAKE_CASE	=				sorted(rules_dict.keys()  )
         for i, sorted_step in enumerate(__A  ):
            if steps < sorted_step:
               return rules_dict[sorted_steps[i]]
         return last_lr_multiple

      return rule_func

   _SCREAMING_SNAKE_CASE	=				create_rules_function(__A	,					__A  )

   return LambdaLR(__A	,					__A	,					last_epoch=__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Union[str, Any]	,					__A    :			Optional[Any]	,					__A    :			List[str]	,					__A    :			Union[str, Any]=-1  )			->			str:

   def lr_lambda(__A    :			int  ):
      if current_step < num_warmup_steps:
         return float(__A  ) / float(max(1	,					__A  )  )
      return max(
          0.0	,					float(num_training_steps - current_step  ) / float(max(1	,					num_training_steps - num_warmup_steps  )  )  )

   return LambdaLR(__A	,					__A	,					__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optimizer	,					__A    :			int	,					__A    :			int	,					__A    :			float = 0.5	,					__A    :			int = -1  )			->			Any:

   def lr_lambda(__A    :			int  ):
      if current_step < num_warmup_steps:
         return float(__A  ) / float(max(1	,					__A  )  )
      _SCREAMING_SNAKE_CASE	=				float(current_step - num_warmup_steps  ) / float(max(1	,					num_training_steps - num_warmup_steps  )  )
      return max(0.0	,					0.5 * (1.0 + math.cos(math.pi * float(__A  ) * 2.0 * progress  ))  )

   return LambdaLR(__A	,					__A	,					__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optimizer	,					__A    :			int	,					__A    :			int	,					__A    :			int = 1	,					__A    :			int = -1  )			->			str:

   def lr_lambda(__A    :			Optional[int]  ):
      if current_step < num_warmup_steps:
         return float(__A  ) / float(max(1	,					__A  )  )
      _SCREAMING_SNAKE_CASE	=				float(current_step - num_warmup_steps  ) / float(max(1	,					num_training_steps - num_warmup_steps  )  )
      if progress >= 1.0:
         return 0.0
      return max(0.0	,					0.5 * (1.0 + math.cos(math.pi * ((float(__A  ) * progress) % 1.0)  ))  )

   return LambdaLR(__A	,					__A	,					__A  )
def 			SCREAMING_SNAKE_CASE_			(							__A    :			Optional[int]	,					__A    :			Optional[Any]	,					__A    :			List[str]	,					__A    :			Any=1e-7	,					__A    :			Optional[Any]=1.0	,					__A    :			List[Any]=-1  )			->			Tuple:
   _SCREAMING_SNAKE_CASE	=				optimizer.defaults["lr"]
   if not (lr_init > lr_end):
      raise ValueError(f"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""  )

   def lr_lambda(__A    :			int  ):
      if current_step < num_warmup_steps:
         return float(__A  ) / float(max(1	,					__A  )  )
      elif current_step > num_training_steps:
         return lr_end / lr_init  # as LambdaLR multiplies by lr_init
      else:
         _SCREAMING_SNAKE_CASE	=				lr_init - lr_end
         _SCREAMING_SNAKE_CASE	=				num_training_steps - num_warmup_steps
         _SCREAMING_SNAKE_CASE	=				1 - (current_step - num_warmup_steps) / decay_steps
         _SCREAMING_SNAKE_CASE	=				lr_range * pct_remaining**power + lr_end
         return decay / lr_init  # as LambdaLR multiplies by lr_init

   return LambdaLR(__A	,					__A	,					__A  )


lowerCamelCase_			=						{
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}






def 			SCREAMING_SNAKE_CASE_			(							__A    :			Union[str, SchedulerType]	,					__A    :			Optimizer	,					__A    :			Optional[str] = None	,					__A    :			Optional[int] = None	,					__A    :			Optional[int] = None	,					__A    :			int = 1	,					__A    :			float = 1.0	,					__A    :			int = -1	,					)			->			Optional[Any]:
   _SCREAMING_SNAKE_CASE	=				SchedulerType(__A  )
   _SCREAMING_SNAKE_CASE	=				TYPE_TO_SCHEDULER_FUNCTION[name]
   if name == SchedulerType.CONSTANT:
      return schedule_func(__A	,					last_epoch=__A  )

   if name == SchedulerType.PIECEWISE_CONSTANT:
      return schedule_func(__A	,					step_rules=__A	,					last_epoch=__A  )

   # All other schedulers require `num_warmup_steps`
   if num_warmup_steps is None:
      raise ValueError(f"""{name} requires `num_warmup_steps`, please provide that argument."""  )

   if name == SchedulerType.CONSTANT_WITH_WARMUP:
      return schedule_func(__A	,					num_warmup_steps=__A	,					last_epoch=__A  )

   # All other schedulers require `num_training_steps`
   if num_training_steps is None:
      raise ValueError(f"""{name} requires `num_training_steps`, please provide that argument."""  )

   if name == SchedulerType.COSINE_WITH_RESTARTS:
      return schedule_func(
          __A	,					num_warmup_steps=__A	,					num_training_steps=__A	,					num_cycles=__A	,					last_epoch=__A	,					)

   if name == SchedulerType.POLYNOMIAL:
      return schedule_func(
          __A	,					num_warmup_steps=__A	,					num_training_steps=__A	,					power=__A	,					last_epoch=__A	,					)

   return schedule_func(
       __A	,					num_warmup_steps=__A	,					num_training_steps=__A	,					last_epoch=__A  )

