

# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce

def       a__					(		UpperCAmelCase							:      Union[str, Any]      )  ->						List[Any]:
  return (torch.arange(state.num_processes      ) + 1.0 + (state.num_processes * state.process_index)).to(state.device      )

def       a__					(		UpperCAmelCase							:      Any      )  ->						int:
  UpperCAmelCase	:							int						=			create_tensor(UpperCAmelCase      )
  UpperCAmelCase	:							Union[str, Any]						=			gather(UpperCAmelCase      )
  assert gathered_tensor.tolist() == list(range(1    ,			state.num_processes**2 + 1      )      )

def       a__					(		UpperCAmelCase							:      int      )  ->						Tuple:
  UpperCAmelCase	:							Optional[int]						=			[state.process_index]
  UpperCAmelCase	:							Union[str, Any]						=			gather_object(UpperCAmelCase      )
  assert len(UpperCAmelCase      ) == state.num_processes, f'''{gathered_obj}, {len(UpperCAmelCase      )} != {state.num_processes}'''
  assert gathered_obj == list(range(state.num_processes      )      ), f'''{gathered_obj} != {list(range(state.num_processes      )      )}'''

def       a__					(		UpperCAmelCase							:      Union[str, Any]      )  ->						Union[str, Any]:
  UpperCAmelCase	:							Tuple						=			create_tensor(UpperCAmelCase      )
  UpperCAmelCase	:							List[str]						=			broadcast(UpperCAmelCase      )
  assert broadcasted_tensor.shape == torch.Size([state.num_processes]      )
  assert broadcasted_tensor.tolist() == list(range(1    ,			state.num_processes + 1      )      )

def       a__					(		UpperCAmelCase							:      Any      )  ->						Tuple:
  # We need to pad the tensor with one more element if we are the main process
  # to ensure that we can pad
  if state.is_main_process:
    UpperCAmelCase	:							Any						=			torch.arange(state.num_processes + 1      ).to(state.device      )
  else:
    UpperCAmelCase	:							int						=			torch.arange(state.num_processes      ).to(state.device      )
  UpperCAmelCase	:							List[str]						=			pad_across_processes(UpperCAmelCase      )
  assert padded_tensor.shape == torch.Size([state.num_processes + 1]      )
  if not state.is_main_process:
    assert padded_tensor.tolist() == list(range(0    ,			state.num_processes      )      ) + [0]

def       a__					(		UpperCAmelCase							:      int      )  ->						List[str]:
  # For now runs on only two processes
  if state.num_processes != 2:
    return
  UpperCAmelCase	:							Tuple						=			create_tensor(UpperCAmelCase      )
  UpperCAmelCase	:							str						=			reduce(UpperCAmelCase    ,			'''sum'''      )
  UpperCAmelCase	:							List[Any]						=			torch.tensor([4.0, 6]      ).to(state.device      )
  assert torch.allclose(UpperCAmelCase    ,			UpperCAmelCase      ), f'''{reduced_tensor} != {truth_tensor}'''

def       a__					(		UpperCAmelCase							:      Tuple      )  ->						int:
  # For now runs on only two processes
  if state.num_processes != 2:
    return
  UpperCAmelCase	:							Dict						=			create_tensor(UpperCAmelCase      )
  UpperCAmelCase	:							List[str]						=			reduce(UpperCAmelCase    ,			'''mean'''      )
  UpperCAmelCase	:							Union[str, Any]						=			torch.tensor([2.0, 3]      ).to(state.device      )
  assert torch.allclose(UpperCAmelCase    ,			UpperCAmelCase      ), f'''{reduced_tensor} != {truth_tensor}'''

def       a__					(		UpperCAmelCase							:      Optional[Any]      )  ->						List[str]:
  # For xla_spawn (TPUs)
  main()

def       a__					(		)  ->						Union[str, Any]:
  UpperCAmelCase	:							List[str]						=			PartialState()
  state.print(f'''State: {state}'''      )
  state.print('''testing gather'''      )
  test_gather(UpperCAmelCase      )
  state.print('''testing gather_object'''      )
  test_gather_object(UpperCAmelCase      )
  state.print('''testing broadcast'''      )
  test_broadcast(UpperCAmelCase      )
  state.print('''testing pad_across_processes'''      )
  test_pad_across_processes(UpperCAmelCase      )
  state.print('''testing reduce_sum'''      )
  test_reduce_sum(UpperCAmelCase      )
  state.print('''testing reduce_mean'''      )
  test_reduce_mean(UpperCAmelCase      )


if __name__ == "__main__":
     main()








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCamelCase    :       str        =							{"configuration_xlnet": ["XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLNetConfig"]}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _lowerCamelCase    :       List[Any]        =							["XLNetTokenizer"]

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _lowerCamelCase    :       Optional[Any]        =							["XLNetTokenizerFast"]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _lowerCamelCase    :       Dict        =							[
         "XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
         "XLNetForMultipleChoice",
         "XLNetForQuestionAnswering",
         "XLNetForQuestionAnsweringSimple",
         "XLNetForSequenceClassification",
         "XLNetForTokenClassification",
         "XLNetLMHeadModel",
         "XLNetModel",
         "XLNetPreTrainedModel",
         "load_tf_weights_in_xlnet",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     _lowerCamelCase    :       Union[str, Any]        =							[
         "TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
         "TFXLNetForMultipleChoice",
         "TFXLNetForQuestionAnsweringSimple",
         "TFXLNetForSequenceClassification",
         "TFXLNetForTokenClassification",
         "TFXLNetLMHeadModel",
         "TFXLNetMainLayer",
         "TFXLNetModel",
         "TFXLNetPreTrainedModel",
     ]


if TYPE_CHECKING:
     from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xlnet import XLNetTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_xlnet_fast import XLNetTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_xlnet import (
              XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
              XLNetForMultipleChoice,
              XLNetForQuestionAnswering,
              XLNetForQuestionAnsweringSimple,
              XLNetForSequenceClassification,
              XLNetForTokenClassification,
              XLNetLMHeadModel,
              XLNetModel,
              XLNetPreTrainedModel,
              load_tf_weights_in_xlnet,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_xlnet import (
              TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFXLNetForMultipleChoice,
              TFXLNetForQuestionAnsweringSimple,
              TFXLNetForSequenceClassification,
              TFXLNetForTokenClassification,
              TFXLNetLMHeadModel,
              TFXLNetMainLayer,
              TFXLNetModel,
              TFXLNetPreTrainedModel,
          )

else:
     import sys

     _lowerCamelCase    :       Optional[Any]        =							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





