



# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
				import botoa  # noqa: F401


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   Any													=					botoa.client("iam"						)

				lowercase__					:   str													=					{
				    "Version": "2012-10-17",
				    "Statement": [
				        {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
				    ],
				}
				try:
								# create the role, associated with the chosen trust policy
								iam_client.create_role(
								    RoleName=lowerCamelCase__					,			AssumeRolePolicyDocument=json.dumps(lowerCamelCase__					,			indent=2						)						)
								lowercase__					:   Tuple													=					{
								    "Version": "2012-10-17",
								    "Statement": [
								        {
								            "Effect": "Allow",
								            "Action": [
								                "sagemaker:*",
								                "ecr:GetDownloadUrlForLayer",
								                "ecr:BatchGetImage",
								                "ecr:BatchCheckLayerAvailability",
								                "ecr:GetAuthorizationToken",
								                "cloudwatch:PutMetricData",
								                "cloudwatch:GetMetricData",
								                "cloudwatch:GetMetricStatistics",
								                "cloudwatch:ListMetrics",
								                "logs:CreateLogGroup",
								                "logs:CreateLogStream",
								                "logs:DescribeLogStreams",
								                "logs:PutLogEvents",
								                "logs:GetLogEvents",
								                "s3:CreateBucket",
								                "s3:ListBucket",
								                "s3:GetBucketLocation",
								                "s3:GetObject",
								                "s3:PutObject",
								            ],
								            "Resource": "*",
								        }
								    ],
								}
								# attach policy to role
								iam_client.put_role_policy(
								    RoleName=lowerCamelCase__					,			PolicyName=F"""{role_name}_policy_permission"""					,			PolicyDocument=json.dumps(lowerCamelCase__					,			indent=2						)					,			)
				except iam_client.exceptions.EntityAlreadyExistsException:
								print(F"""role {role_name} already exists. Using existing one"""						)


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   List[Any]													=					botoa.client("iam"						)
				return iam_client.get_role(RoleName=lowerCamelCase__						)["Role"]["Arn"]


def  __lowerCamelCase (       ):

				"""simple docstring"""


				lowercase__					:   str													=					_ask_options(
				    "How do you want to authorize?"					,			["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "]					,			lowerCamelCase__					,			)
				lowercase__					:   Any													=					None
				if credentials_configuration == 0:
								lowercase__					:   int													=					_ask_field("Enter your AWS Profile name: [default] "					,			default="default"						)
								lowercase__					:   Tuple													=					aws_profile
				else:
								print(
								    "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"
								    "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"						)
								lowercase__					:   Any													=					_ask_field("AWS Access Key ID: "						)
								lowercase__					:   List[Any]													=					aws_access_key_id

								lowercase__					:   int													=					_ask_field("AWS Secret Access Key: "						)
								lowercase__					:   Optional[Any]													=					aws_secret_access_key

				lowercase__					:   Any													=					_ask_field("Enter your AWS Region: [us-east-1]"					,			default="us-east-1"						)
				lowercase__					:   Union[str, Any]													=					aws_region

				lowercase__					:   Tuple													=					_ask_options(
				    "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"					,			["Provide IAM Role name", "Create new IAM role using credentials"]					,			lowerCamelCase__					,			)
				if role_management == 0:
								lowercase__					:   Tuple													=					_ask_field("Enter your IAM role name: "						)
				else:
								lowercase__					:   Optional[int]													=					"accelerate_sagemaker_execution_role"
								print(F"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""						)
								_create_iam_role_for_sagemaker(lowerCamelCase__						)

				lowercase__					:   int													=					_ask_field(
				    "Do you want to use custom Docker image? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
				lowercase__					:   Any													=					None
				if is_custom_docker_image:
								lowercase__					:   Tuple													=					_ask_field("Enter your Docker image: "					,			lambda lowerCamelCase__						: str(lowerCamelCase__						).lower()						)

				lowercase__					:   Tuple													=					_ask_field(
				    "Do you want to provide SageMaker input channels with data locations? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
				lowercase__					:   str													=					None
				if is_sagemaker_inputs_enabled:
								lowercase__					:   str													=					_ask_field(
								    "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): "					,			lambda lowerCamelCase__						: str(lowerCamelCase__						).lower()					,			)

				lowercase__					:   Tuple													=					_ask_field(
				    "Do you want to enable SageMaker metrics? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
				lowercase__					:   List[str]													=					None
				if is_sagemaker_metrics_enabled:
								lowercase__					:   Dict													=					_ask_field(
								    "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): "					,			lambda lowerCamelCase__						: str(lowerCamelCase__						).lower()					,			)

				lowercase__					:   int													=					_ask_options(
				    "What is the distributed mode?"					,			["No distributed training", "Data parallelism"]					,			_convert_sagemaker_distributed_mode					,			)
				lowercase__					:   Union[str, Any]													=					{}
				lowercase__					:   Union[str, Any]													=					_ask_field(
				    "Do you wish to optimize your script with torch dynamo?[yes/NO]:"					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
				if use_dynamo:
								lowercase__					:   int													=					"dynamo_"
								lowercase__					:   Optional[int]													=					_ask_options(
								    "Which dynamo backend would you like to use?"					,			[x.lower() for x in DYNAMO_BACKENDS]					,			_convert_dynamo_backend					,			default=2					,			)
								lowercase__					:   Optional[Any]													=					_ask_field(
								    "Do you want to customize the defaults sent to torch.compile? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)

								if use_custom_options:
												lowercase__					:   int													=					_ask_options(
												    "Which mode do you want to use?"					,			lowerCamelCase__					,			lambda lowerCamelCase__						: TORCH_DYNAMO_MODES[int(lowerCamelCase__						)]					,			default="default"					,			)
												lowercase__					:   Any													=					_ask_field(
												    "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
												lowercase__					:   Optional[int]													=					_ask_field(
												    "Do you want to enable dynamic shape tracing? [yes/NO]: "					,			_convert_yes_no_to_bool					,			default=lowerCamelCase__					,			error_message="Please enter yes or no."					,			)
				lowercase__					:   int													=					"Which EC2 instance type you want to use for your training?"
				if distributed_type != SageMakerDistributedType.NO:
								lowercase__					:   str													=					_ask_options(
								    lowerCamelCase__					,			lowerCamelCase__					,			lambda lowerCamelCase__						: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(lowerCamelCase__						)]						)
				else:
								eca_instance_query += "? [ml.p3.2xlarge]:"
								lowercase__					:   Union[str, Any]													=					_ask_field(lowerCamelCase__					,			lambda lowerCamelCase__						: str(lowerCamelCase__						).lower()					,			default="ml.p3.2xlarge"						)

				lowercase__					:   Dict													=					1
				if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
								lowercase__					:   Optional[int]													=					_ask_field(
								    "How many machines do you want use? [1]: "					,			lowerCamelCase__					,			default=1					,			)

				lowercase__					:   Union[str, Any]													=					_ask_options(
				    "Do you wish to use FP16 or BF16 (mixed precision)?"					,			["no", "fp16", "bf16", "fp8"]					,			_convert_mixed_precision					,			)

				if use_dynamo and mixed_precision == "no":
								print(
								    "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."						)

				return SageMakerConfig(
				    image_uri=lowerCamelCase__					,			compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER					,			distributed_type=lowerCamelCase__					,			use_cpu=lowerCamelCase__					,			dynamo_config=lowerCamelCase__					,			eca_instance_type=lowerCamelCase__					,			profile=lowerCamelCase__					,			region=lowerCamelCase__					,			iam_role_name=lowerCamelCase__					,			mixed_precision=lowerCamelCase__					,			num_machines=lowerCamelCase__					,			sagemaker_inputs_file=lowerCamelCase__					,			sagemaker_metrics_file=lowerCamelCase__					,			)









import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__					  =  logging.get_logger(__name__)

lowerCAmelCase__					  =  '''▁'''

lowerCAmelCase__					  =  {'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

lowerCAmelCase__					  =  {
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

lowerCAmelCase__					  =  {'''vinai/bartpho-syllable''': 1_0_2_4}

class 				snake_case__(_UpperCamelCase      ):

					"""simple docstring"""






					lowercase_								=		VOCAB_FILES_NAMES
					lowercase_								=		PRETRAINED_VOCAB_FILES_MAP
					lowercase_								=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase_								=		["""input_ids""", """attention_mask"""]







					def __init__(							self :     Optional[int]				,       SCREAMING_SNAKE_CASE :     List[str]				,       SCREAMING_SNAKE_CASE :     Optional[Any]				,       SCREAMING_SNAKE_CASE :     Optional[int]="<s>"				,       SCREAMING_SNAKE_CASE :     Optional[int]="</s>"				,       SCREAMING_SNAKE_CASE :     str="</s>"				,       SCREAMING_SNAKE_CASE :     List[str]="<s>"				,       SCREAMING_SNAKE_CASE :     Union[str, Any]="<unk>"				,       SCREAMING_SNAKE_CASE :     Tuple="<pad>"				,       SCREAMING_SNAKE_CASE :     List[str]="<mask>"				,       SCREAMING_SNAKE_CASE :     Optional[Dict[str, Any]] = None				,       **SCREAMING_SNAKE_CASE :     int				,       ):
									# Mask token behave like a normal word, i.e. include the space before it
									lowercase__					:   Dict													=					AddedToken(SCREAMING_SNAKE_CASE				,       lstrip=SCREAMING_SNAKE_CASE				,       rstrip=SCREAMING_SNAKE_CASE					) if isinstance(SCREAMING_SNAKE_CASE				,       SCREAMING_SNAKE_CASE					) else mask_token

									lowercase__					:   int													=					{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=SCREAMING_SNAKE_CASE				,       eos_token=SCREAMING_SNAKE_CASE				,       unk_token=SCREAMING_SNAKE_CASE				,       sep_token=SCREAMING_SNAKE_CASE				,       cls_token=SCREAMING_SNAKE_CASE				,       pad_token=SCREAMING_SNAKE_CASE				,       mask_token=SCREAMING_SNAKE_CASE				,       sp_model_kwargs=self.sp_model_kwargs				,       **SCREAMING_SNAKE_CASE				,       )

									lowercase__					:   Dict													=					vocab_file
									lowercase__					:   Union[str, Any]													=					monolingual_vocab_file
									lowercase__					:   Union[str, Any]													=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(str(SCREAMING_SNAKE_CASE					)					)

									# Load the reduced vocab

									# Keep order of special tokens for backward compatibility
									lowercase__					:   Any													=					{}
									lowercase__					:   int													=					0
									for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
													if str(SCREAMING_SNAKE_CASE					) not in self.fairseq_tokens_to_ids:
																	lowercase__					:   Dict													=					cnt
																	cnt += 1
									with open(SCREAMING_SNAKE_CASE				,       "r"				,       encoding="utf-8"					) as f:
													for line in f.readlines():
																	lowercase__					:   int													=					line.strip().split()[0]
																	lowercase__					:   List[str]													=					len(self.fairseq_tokens_to_ids					)
									if str(SCREAMING_SNAKE_CASE					) not in self.fairseq_tokens_to_ids:
													lowercase__					:   Optional[Any]													=					len(self.fairseq_tokens_to_ids					)

									lowercase__					:   List[str]													=					{v: k for k, v in self.fairseq_tokens_to_ids.items()}







					def __getstate__(							self :     int					):
									lowercase__					:   Dict													=					self.__dict__.copy()
									lowercase__					:   Union[str, Any]													=					None
									lowercase__					:   Dict													=					self.sp_model.serialized_model_proto()
									return state







					def __setstate__(							self :     Union[str, Any]				,       SCREAMING_SNAKE_CASE :     Optional[Any]					):
									lowercase__					:   Union[str, Any]													=					d

									# for backward compatibility
									if not hasattr(self				,       "sp_model_kwargs"					):
													lowercase__					:   Dict													=					{}

									lowercase__					:   List[str]													=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto					)







					def 			snake_case					(							self :     Optional[int]				,       SCREAMING_SNAKE_CASE :     List[int]				,       SCREAMING_SNAKE_CASE :     Optional[List[int]] = None					):

									if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									lowercase__					:   List[Any]													=					[self.cls_token_id]
									lowercase__					:   str													=					[self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep







					def 			snake_case					(							self :     Dict				,       SCREAMING_SNAKE_CASE :     List[int]				,       SCREAMING_SNAKE_CASE :     Optional[List[int]] = None				,       SCREAMING_SNAKE_CASE :     bool = False					):

									if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=SCREAMING_SNAKE_CASE				,       token_ids_a=SCREAMING_SNAKE_CASE				,       already_has_special_tokens=SCREAMING_SNAKE_CASE					)

									if token_ids_a is None:
													return [1] + ([0] * len(SCREAMING_SNAKE_CASE					)) + [1]
									return [1] + ([0] * len(SCREAMING_SNAKE_CASE					)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE					)) + [1]







					def 			snake_case					(							self :     str				,       SCREAMING_SNAKE_CASE :     List[int]				,       SCREAMING_SNAKE_CASE :     Optional[List[int]] = None					):
									lowercase__					:   Tuple													=					[self.sep_token_id]
									lowercase__					:   str													=					[self.cls_token_id]

									if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







					@property
					def 			snake_case					(							self :     Optional[int]					):
									return len(self.fairseq_ids_to_tokens					)







					def 			snake_case					(							self :     List[Any]					):
									lowercase__					:   Any													=					{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab







					def 			snake_case					(							self :     Union[str, Any]				,       SCREAMING_SNAKE_CASE :     str					):
									return self.sp_model.encode(SCREAMING_SNAKE_CASE				,       out_type=SCREAMING_SNAKE_CASE					)







					def 			snake_case					(							self :     List[str]				,       SCREAMING_SNAKE_CASE :     int					):
									if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
									else:
													return self.unk_token_id







					def 			snake_case					(							self :     List[str]				,       SCREAMING_SNAKE_CASE :     Any					):
									return self.fairseq_ids_to_tokens[index]







					def 			snake_case					(							self :     List[Any]				,       SCREAMING_SNAKE_CASE :     Union[str, Any]					):
									lowercase__					:   str													=					"".join(SCREAMING_SNAKE_CASE					).replace(SCREAMING_SNAKE_CASE				,       " "					).strip()
									return out_string







					def 			snake_case					(							self :     Optional[int]				,       SCREAMING_SNAKE_CASE :     str				,       SCREAMING_SNAKE_CASE :     Optional[str] = None					):
									if not os.path.isdir(SCREAMING_SNAKE_CASE					):
													logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
													return
									lowercase__					:   str													=					os.path.join(
									    SCREAMING_SNAKE_CASE				,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
									lowercase__					:   List[str]													=					os.path.join(
									    SCREAMING_SNAKE_CASE				,       (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]				,       )

									if os.path.abspath(self.vocab_file					) != os.path.abspath(SCREAMING_SNAKE_CASE					) and os.path.isfile(self.vocab_file					):
													copyfile(self.vocab_file				,       SCREAMING_SNAKE_CASE					)
									elif not os.path.isfile(self.vocab_file					):
													with open(SCREAMING_SNAKE_CASE				,       "wb"					) as fi:
																	lowercase__					:   str													=					self.sp_model.serialized_model_proto()
																	fi.write(SCREAMING_SNAKE_CASE					)

									if os.path.abspath(self.monolingual_vocab_file					) != os.path.abspath(
									    SCREAMING_SNAKE_CASE					) and os.path.isfile(self.monolingual_vocab_file					):
													copyfile(self.monolingual_vocab_file				,       SCREAMING_SNAKE_CASE					)
									elif not os.path.isfile(self.monolingual_vocab_file					):
													with open(SCREAMING_SNAKE_CASE				,       "w"				,       encoding="utf-8"					) as fp:
																	for token in self.fairseq_tokens_to_ids:
																					if token not in self.all_special_tokens:
																									fp.write(f"""{str(SCREAMING_SNAKE_CASE					)} \n"""					)

									return out_vocab_file, out_monolingual_vocab_file




