

'''simple docstring'''
import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
     from PIL import Image

     from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
     import torch

if is_tf_available():
     import tensorflow as tf







@require_vision
@require_torchvision
class 							A__   (	unittest.TestCase  ):





  def 						A  (		self				:     Tuple			)     ->   Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE							   =SamImageProcessor()
       _SCREAMING_SNAKE_CASE							   =SamProcessor(_a			)
       processor.save_pretrained(self.tmpdirname			)





  def 						A  (		self				:     Tuple							, **_a				:     Any			)     ->   List[Any]:





       '''simple docstring'''






       return AutoProcessor.from_pretrained(self.tmpdirname							, **_a			).image_processor





  def 						A  (		self				:     Dict			)     ->   int:





       '''simple docstring'''






       shutil.rmtree(self.tmpdirname			)





  def 						A  (		self				:     Any			)     ->   Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =[np.random.randint(255							, size=(3, 30, 400)							, dtype=np.uinta			)]

       _SCREAMING_SNAKE_CASE							   =[Image.fromarray(np.moveaxis(_a							, 0							, -1			)			) for x in image_inputs]

       return image_inputs





  def 						A  (		self				:     Any			)     ->   Tuple:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=self.get_image_processor()			)
       processor.save_pretrained(self.tmpdirname			)

       _SCREAMING_SNAKE_CASE							   =self.get_image_processor(do_normalize=_a							, padding_value=1.0			)

       _SCREAMING_SNAKE_CASE							   =SamProcessor.from_pretrained(self.tmpdirname							, do_normalize=_a							, padding_value=1.0			)

       self.assertEqual(processor.image_processor.to_json_string()							, image_processor_add_kwargs.to_json_string()			)
       self.assertIsInstance(processor.image_processor							, _a			)





  def 						A  (		self				:     Union[str, Any]			)     ->   List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)

       _SCREAMING_SNAKE_CASE							   =self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE							   =image_processor(_a							, return_tensors='np'			)
       _SCREAMING_SNAKE_CASE							   =processor(images=_a							, return_tensors='np'			)

       input_feat_extract.pop('original_sizes'			)  # pop original_sizes as it is popped in the processor
       input_feat_extract.pop('reshaped_input_sizes'			)  # pop original_sizes as it is popped in the processor

       for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum()							, input_processor[key].sum()							, delta=1e-2			)





  @require_torch
  def 						A  (		self				:     int			)     ->   str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)
       _SCREAMING_SNAKE_CASE							   =[torch.ones((1, 3, 5, 5)			)]

       _SCREAMING_SNAKE_CASE							   =[[1764, 2646]]

       _SCREAMING_SNAKE_CASE							   =[[683, 1024]]
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(_a							, _a							, _a			)
       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
           _a							, torch.tensor(_a			)							, torch.tensor(_a			)			)
       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       # should also work with np
       _SCREAMING_SNAKE_CASE							   =[np.ones((1, 3, 5, 5)			)]
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(_a							, np.array(_a			)							, np.array(_a			)			)

       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       _SCREAMING_SNAKE_CASE							   =[[1, 0], [0, 1]]
       with self.assertRaises(_a			):
            _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(_a							, np.array(_a			)							, np.array(_a			)			)







@require_vision
@require_tf
class 							A__   (	unittest.TestCase  ):





  def 						A  (		self				:     Optional[int]			)     ->   str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE							   =SamImageProcessor()
       _SCREAMING_SNAKE_CASE							   =SamProcessor(_a			)
       processor.save_pretrained(self.tmpdirname			)





  def 						A  (		self				:     List[Any]							, **_a				:     Any			)     ->   Dict:





       '''simple docstring'''






       return AutoProcessor.from_pretrained(self.tmpdirname							, **_a			).image_processor





  def 						A  (		self				:     List[Any]			)     ->   Optional[Any]:





       '''simple docstring'''






       shutil.rmtree(self.tmpdirname			)





  def 						A  (		self				:     str			)     ->   List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =[np.random.randint(255							, size=(3, 30, 400)							, dtype=np.uinta			)]

       _SCREAMING_SNAKE_CASE							   =[Image.fromarray(np.moveaxis(_a							, 0							, -1			)			) for x in image_inputs]

       return image_inputs





  def 						A  (		self				:     Optional[Any]			)     ->   Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=self.get_image_processor()			)
       processor.save_pretrained(self.tmpdirname			)

       _SCREAMING_SNAKE_CASE							   =self.get_image_processor(do_normalize=_a							, padding_value=1.0			)

       _SCREAMING_SNAKE_CASE							   =SamProcessor.from_pretrained(self.tmpdirname							, do_normalize=_a							, padding_value=1.0			)

       self.assertEqual(processor.image_processor.to_json_string()							, image_processor_add_kwargs.to_json_string()			)
       self.assertIsInstance(processor.image_processor							, _a			)





  def 						A  (		self				:     str			)     ->   Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)

       _SCREAMING_SNAKE_CASE							   =self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE							   =image_processor(_a							, return_tensors='np'			)
       _SCREAMING_SNAKE_CASE							   =processor(images=_a							, return_tensors='np'			)

       input_feat_extract.pop('original_sizes'			)  # pop original_sizes as it is popped in the processor
       input_feat_extract.pop('reshaped_input_sizes'			)  # pop reshaped_input_sizes as it is popped in the processor

       for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum()							, input_processor[key].sum()							, delta=1e-2			)





  @require_tf
  def 						A  (		self				:     Dict			)     ->   Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)
       _SCREAMING_SNAKE_CASE							   =[tf.ones((1, 3, 5, 5)			)]

       _SCREAMING_SNAKE_CASE							   =[[1764, 2646]]

       _SCREAMING_SNAKE_CASE							   =[[683, 1024]]
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(_a							, _a							, _a							, return_tensors='tf'			)
       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
           _a							, tf.convert_to_tensor(_a			)							, tf.convert_to_tensor(_a			)							, return_tensors='tf'							, )
       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       # should also work with np
       _SCREAMING_SNAKE_CASE							   =[np.ones((1, 3, 5, 5)			)]
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
           _a							, np.array(_a			)							, np.array(_a			)							, return_tensors='tf'			)

       self.assertEqual(masks[0].shape							, (1, 3, 1764, 2646)			)

       _SCREAMING_SNAKE_CASE							   =[[1, 0], [0, 1]]
       with self.assertRaises(tf.errors.InvalidArgumentError			):
            _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
                _a							, np.array(_a			)							, np.array(_a			)							, return_tensors='tf'			)







@require_vision
@require_torchvision
class 							A__   (	unittest.TestCase  ):





  def 						A  (		self				:     int			)     ->   Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =tempfile.mkdtemp()
       _SCREAMING_SNAKE_CASE							   =SamImageProcessor()
       _SCREAMING_SNAKE_CASE							   =SamProcessor(_a			)
       processor.save_pretrained(self.tmpdirname			)





  def 						A  (		self				:     Union[str, Any]							, **_a				:     Tuple			)     ->   Optional[int]:





       '''simple docstring'''






       return AutoProcessor.from_pretrained(self.tmpdirname							, **_a			).image_processor





  def 						A  (		self				:     str			)     ->   Optional[int]:





       '''simple docstring'''






       shutil.rmtree(self.tmpdirname			)





  def 						A  (		self				:     List[str]			)     ->   Tuple:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =[np.random.randint(255							, size=(3, 30, 400)							, dtype=np.uinta			)]

       _SCREAMING_SNAKE_CASE							   =[Image.fromarray(np.moveaxis(_a							, 0							, -1			)			) for x in image_inputs]

       return image_inputs





  @is_pt_tf_cross_test
  def 						A  (		self				:     Any			)     ->   List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)
       _SCREAMING_SNAKE_CASE							   =np.random.randint(0							, 2							, size=(1, 3, 5, 5)			).astype(np.floataa			)
       _SCREAMING_SNAKE_CASE							   =[tf.convert_to_tensor(_a			)]
       _SCREAMING_SNAKE_CASE							   =[torch.tensor(_a			)]

       _SCREAMING_SNAKE_CASE							   =[[1764, 2646]]

       _SCREAMING_SNAKE_CASE							   =[[683, 1024]]
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
           _a							, _a							, _a							, return_tensors='tf'			)
       _SCREAMING_SNAKE_CASE							   =processor.post_process_masks(
           _a							, _a							, _a							, return_tensors='pt'			)

       self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()			)			)





  @is_pt_tf_cross_test
  def 						A  (		self				:     Optional[Any]			)     ->   Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							   =self.get_image_processor()

       _SCREAMING_SNAKE_CASE							   =SamProcessor(image_processor=_a			)

       _SCREAMING_SNAKE_CASE							   =self.prepare_image_inputs()

       _SCREAMING_SNAKE_CASE							   =image_processor(_a							, return_tensors='pt'			)['pixel_values'].numpy()
       _SCREAMING_SNAKE_CASE							   =processor(images=_a							, return_tensors='pt'			)['pixel_values'].numpy()

       _SCREAMING_SNAKE_CASE							   =image_processor(_a							, return_tensors='tf'			)['pixel_values'].numpy()
       _SCREAMING_SNAKE_CASE							   =processor(images=_a							, return_tensors='tf'			)['pixel_values'].numpy()

       self.assertTrue(np.allclose(_a							, _a			)			)
       self.assertTrue(np.allclose(_a							, _a			)			)
       self.assertTrue(np.allclose(_a							, _a			)			)



'''simple docstring'''
import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
lowerCamelCase			:				int 			=					logging.getLogger(__name__)

if __name__ == "__main__":
     lowerCamelCase			:				Optional[Any] 			=					argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0_5_2_2, type=int)
     lowerCamelCase			:				Optional[Any] 			=					parser.parse_args()

     logger.info(f'''Loading data from {args.data_file}''')
     with open(args.data_file, "rb") as fp:
          lowerCamelCase			:				Optional[int] 			=					pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     lowerCamelCase			:				Dict 			=					Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     lowerCamelCase			:				Tuple 			=					[0] * args.vocab_size
     for k, v in counter.items():
          lowerCamelCase			:				Any 			=					v

     logger.info(f'''Dump to {args.token_counts_dump}''')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)
