




import argparse
import os
import re

import packaging.version


UpperCamelCase__         =      """examples/"""
UpperCamelCase__         =      {
    """examples""": (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R"""\1version=\"VERSION\","""),
    """doc""": (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
UpperCamelCase__         =      {
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
UpperCamelCase__         =      """README.md"""







def      _a				(						SCREAMING_SNAKE_CASE_							:    Any				,				SCREAMING_SNAKE_CASE_							:    str				,				SCREAMING_SNAKE_CASE_							:    Optional[int]					):
					with open(SCREAMING_SNAKE_CASE_				,				"r"				,				encoding="utf-8"				,				newline="\n"					) as f:
										__lowerCAmelCase           =	f.read()
					__lowerCAmelCase       ,							__lowerCAmelCase           =	REPLACE_PATTERNS[pattern]
					__lowerCAmelCase           =	replace.replace("VERSION"				,				SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase           =	re_pattern.sub(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
					with open(SCREAMING_SNAKE_CASE_				,				"w"				,				encoding="utf-8"				,				newline="\n"					) as f:
										f.write(SCREAMING_SNAKE_CASE_					)







def      _a				(						SCREAMING_SNAKE_CASE_							:    List[str]					):
					for folder, directories, fnames in os.walk(SCREAMING_SNAKE_CASE_					):
										# Removing some of the folders with non-actively maintained examples from the walk
										if "research_projects" in directories:
															directories.remove("research_projects"					)
										if "legacy" in directories:
															directories.remove("legacy"					)
										for fname in fnames:
															if fname.endswith(".py"					):
																				update_version_in_file(os.path.join(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)				,				SCREAMING_SNAKE_CASE_				,				pattern="examples"					)







def      _a				(						SCREAMING_SNAKE_CASE_							:    Tuple				,				SCREAMING_SNAKE_CASE_							:    Union[str, Any]=False					):
					for pattern, fname in REPLACE_FILES.items():
										update_version_in_file(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
					if not patch:
										update_version_in_examples(SCREAMING_SNAKE_CASE_					)







def      _a				(						):
					__lowerCAmelCase           =	"🤗 Transformers currently provides the following architectures"
					__lowerCAmelCase           =	"1. Want to contribute a new model?"
					with open(SCREAMING_SNAKE_CASE_				,				"r"				,				encoding="utf-8"				,				newline="\n"					) as f:
										__lowerCAmelCase           =	f.readlines()

					# Find the start of the list.
					__lowerCAmelCase           =	0
					while not lines[start_index].startswith(_start_prompt					):
										start_index += 1
					start_index += 1

					__lowerCAmelCase           =	start_index
					# Update the lines in the model list.
					while not lines[index].startswith(_end_prompt					):
										if lines[index].startswith("1."					):
															__lowerCAmelCase           =	lines[index].replace(
															    "https://huggingface.co/docs/transformers/main/model_doc"				,				"https://huggingface.co/docs/transformers/model_doc"				,				)
										index += 1

					with open(SCREAMING_SNAKE_CASE_				,				"w"				,				encoding="utf-8"				,				newline="\n"					) as f:
										f.writelines(SCREAMING_SNAKE_CASE_					)







def      _a				(						):
					with open(REPLACE_FILES["init"]				,				"r"					) as f:
										__lowerCAmelCase           =	f.read()
					__lowerCAmelCase           =	REPLACE_PATTERNS["init"][0].search(SCREAMING_SNAKE_CASE_					).groups()[0]
					return packaging.version.parse(SCREAMING_SNAKE_CASE_					)







def      _a				(						SCREAMING_SNAKE_CASE_							:    Optional[Any]=False					):
					__lowerCAmelCase           =	get_version()
					if patch and default_version.is_devrelease:
										raise ValueError("Can't create a patch version from the dev branch, checkout a released version!"					)
					if default_version.is_devrelease:
										__lowerCAmelCase           =	default_version.base_version
					elif patch:
										__lowerCAmelCase           =	F"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
					else:
										__lowerCAmelCase           =	F"""{default_version.major}.{default_version.minor + 1}.0"""

					# Now let's ask nicely if that's the right one.
					__lowerCAmelCase           =	input(F"""Which version are you releasing? [{default_version}]"""					)
					if len(SCREAMING_SNAKE_CASE_					) == 0:
										__lowerCAmelCase           =	default_version

					print(F"""Updating version to {version}."""					)
					global_version_update(SCREAMING_SNAKE_CASE_				,				patch=SCREAMING_SNAKE_CASE_					)
					if not patch:
										print("Cleaning main README, don't forget to run `make fix-copies`."					)
										clean_main_ref_in_model_list()







def      _a				(						):
					__lowerCAmelCase           =	get_version()
					__lowerCAmelCase           =	F"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
					__lowerCAmelCase           =	current_version.base_version

					# Check with the user we got that right.
					__lowerCAmelCase           =	input(F"""Which version are we developing now? [{dev_version}]"""					)
					if len(SCREAMING_SNAKE_CASE_					) == 0:
										__lowerCAmelCase           =	dev_version

					print(F"""Updating version to {version}."""					)
					global_version_update(SCREAMING_SNAKE_CASE_					)
					print("Cleaning main README, don't forget to run `make fix-copies`."					)
					clean_main_ref_in_model_list()


if __name__ == "__main__":
					UpperCamelCase__         =      argparse.ArgumentParser()
					parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
					parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
					UpperCamelCase__         =      parser.parse_args()
					if not args.post_release:
										pre_release_work(patch=args.patch)
					elif args.patch:
										print("""Nothing to do after a patch :-)""")
					else:
										post_release_work()










import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCamelCase__         =      16
UpperCamelCase__         =      32







def      _a				(						SCREAMING_SNAKE_CASE_							:    Accelerator				,				SCREAMING_SNAKE_CASE_							:    int = 16					):
					__lowerCAmelCase           =	AutoTokenizer.from_pretrained("bert-base-cased"					)
					__lowerCAmelCase           =	load_dataset("glue"				,				"mrpc"					)

					def tokenize_function(SCREAMING_SNAKE_CASE_							:    str					):
										# max_length=None => use the model max length (it's actually the default)
										__lowerCAmelCase           =	tokenizer(examples["sentence1"]				,				examples["sentence2"]				,				truncation=SCREAMING_SNAKE_CASE_				,				max_length=SCREAMING_SNAKE_CASE_					)
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										__lowerCAmelCase           =	datasets.map(
										    SCREAMING_SNAKE_CASE_				,				batched=SCREAMING_SNAKE_CASE_				,				remove_columns=["idx", "sentence1", "sentence2"]				,				)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					__lowerCAmelCase           =	tokenized_datasets.rename_column("label"				,				"labels"					)

					def collate_fn(SCREAMING_SNAKE_CASE_							:    int					):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										__lowerCAmelCase           =	1_28 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															__lowerCAmelCase           =	16
										elif accelerator.mixed_precision != "no":
															__lowerCAmelCase           =	8
										else:
															__lowerCAmelCase           =	None

										return tokenizer.pad(
										    SCREAMING_SNAKE_CASE_				,				padding="longest"				,				max_length=SCREAMING_SNAKE_CASE_				,				pad_to_multiple_of=SCREAMING_SNAKE_CASE_				,				return_tensors="pt"				,				)

					# Instantiate dataloaders.
					__lowerCAmelCase           =	DataLoader(
					    tokenized_datasets["train"]				,				shuffle=SCREAMING_SNAKE_CASE_				,				collate_fn=SCREAMING_SNAKE_CASE_				,				batch_size=SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase           =	DataLoader(
					    tokenized_datasets["validation"]				,				shuffle=SCREAMING_SNAKE_CASE_				,				collate_fn=SCREAMING_SNAKE_CASE_				,				batch_size=SCREAMING_SNAKE_CASE_					)

					return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
					from accelerate.test_utils.training import mocked_dataloaders

					UpperCamelCase__         =      mocked_dataloaders  # noqa: F811







def      _a				(						SCREAMING_SNAKE_CASE_							:    str				,				SCREAMING_SNAKE_CASE_							:    str					):
					# For testing only
					if os.environ.get("TESTING_MOCKED_DATALOADERS"				,				SCREAMING_SNAKE_CASE_					) == "1":
										__lowerCAmelCase           =	2
					# New Code #
					__lowerCAmelCase           =	int(args.gradient_accumulation_steps					)
					# Initialize accelerator
					__lowerCAmelCase           =	Accelerator(
					    cpu=args.cpu				,				mixed_precision=args.mixed_precision				,				gradient_accumulation_steps=SCREAMING_SNAKE_CASE_					)
					if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
										raise NotImplementedError(
										    "Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`"					)
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					__lowerCAmelCase           =	config["lr"]
					__lowerCAmelCase           =	int(config["num_epochs"]					)
					__lowerCAmelCase           =	int(config["seed"]					)
					__lowerCAmelCase           =	int(config["batch_size"]					)

					__lowerCAmelCase           =	evaluate.load("glue"				,				"mrpc"					)

					set_seed(SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase       ,							__lowerCAmelCase           =	get_dataloaders(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					__lowerCAmelCase           =	AutoModelForSequenceClassification.from_pretrained("bert-base-cased"				,				return_dict=SCREAMING_SNAKE_CASE_					)

					# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
					# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
					# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
					__lowerCAmelCase           =	model.to(accelerator.device					)

					# Instantiate optimizer
					__lowerCAmelCase           =	AdamW(params=model.parameters()				,				lr=SCREAMING_SNAKE_CASE_					)

					# Instantiate scheduler
					__lowerCAmelCase           =	get_linear_schedule_with_warmup(
					    optimizer=SCREAMING_SNAKE_CASE_				,				num_warmup_steps=1_00				,				num_training_steps=(len(SCREAMING_SNAKE_CASE_					) * num_epochs)				,				)

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase           =	accelerator.prepare(
					    SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)

					# Now we train the model
					for epoch in range(SCREAMING_SNAKE_CASE_					):
										model.train()
										for step, batch in enumerate(SCREAMING_SNAKE_CASE_					):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device					)
															# New code #
															# We use the new `accumulate` context manager to perform gradient accumulation
															# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
															with accelerator.accumulate(SCREAMING_SNAKE_CASE_					):
																				__lowerCAmelCase           =	model(**SCREAMING_SNAKE_CASE_					)
																				__lowerCAmelCase           =	output.loss
																				accelerator.backward(SCREAMING_SNAKE_CASE_					)
																				optimizer.step()
																				lr_scheduler.step()
																				optimizer.zero_grad()

										model.eval()
										for step, batch in enumerate(SCREAMING_SNAKE_CASE_					):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device					)
															with torch.no_grad():
																				__lowerCAmelCase           =	model(**SCREAMING_SNAKE_CASE_					)
															__lowerCAmelCase           =	outputs.logits.argmax(dim=-1					)
															__lowerCAmelCase       ,							__lowerCAmelCase           =	accelerator.gather_for_metrics((predictions, batch["labels"])					)
															metric.add_batch(
															    predictions=SCREAMING_SNAKE_CASE_				,				references=SCREAMING_SNAKE_CASE_				,				)

										__lowerCAmelCase           =	metric.compute()
										# Use accelerator.print to print only on the main process.
										accelerator.print(F"""epoch {epoch}:"""				,				SCREAMING_SNAKE_CASE_					)







def      _a				(						):
					__lowerCAmelCase           =	argparse.ArgumentParser(description="Simple example of training script."					)
					parser.add_argument(
					    "--mixed_precision"				,				type=SCREAMING_SNAKE_CASE_				,				default=SCREAMING_SNAKE_CASE_				,				choices=["no", "fp16", "bf16", "fp8"]				,				help="Whether to use mixed precision. Choose"
					    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
					    "and an Nvidia Ampere GPU."				,				)
					# New Code #
					parser.add_argument(
					    "--gradient_accumulation_steps"				,				type=SCREAMING_SNAKE_CASE_				,				default=1				,				help="The number of minibatches to be ran before gradients are accumulated."				,				)
					parser.add_argument("--cpu"				,				action="store_true"				,				help="If passed, will train on the CPU."					)
					__lowerCAmelCase           =	parser.parse_args()
					__lowerCAmelCase           =	{"lr": 2E-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
					training_function(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)


if __name__ == "__main__":
					main()




