


"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__snake_case             =   {
    """configuration_clip""": [
        """CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """CLIPConfig""",
        """CLIPOnnxConfig""",
        """CLIPTextConfig""",
        """CLIPVisionConfig""",
    ],
    """processing_clip""": ["""CLIPProcessor"""],
    """tokenization_clip""": ["""CLIPTokenizer"""],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case             =   ["""CLIPTokenizerFast"""]

try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case             =   ["""CLIPFeatureExtractor"""]
  __snake_case             =   ["""CLIPImageProcessor"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case             =   [
      """CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """CLIPModel""",
      """CLIPPreTrainedModel""",
      """CLIPTextModel""",
      """CLIPTextModelWithProjection""",
      """CLIPVisionModel""",
      """CLIPVisionModelWithProjection""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case             =   [
      """TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFCLIPModel""",
      """TFCLIPPreTrainedModel""",
      """TFCLIPTextModel""",
      """TFCLIPVisionModel""",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case             =   [
      """FlaxCLIPModel""",
      """FlaxCLIPPreTrainedModel""",
      """FlaxCLIPTextModel""",
      """FlaxCLIPTextPreTrainedModel""",
      """FlaxCLIPVisionModel""",
      """FlaxCLIPVisionPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_clip import (
      CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
      CLIPConfig,
      CLIPOnnxConfig,
      CLIPTextConfig,
      CLIPVisionConfig,
  )
  from .processing_clip import CLIPProcessor
  from .tokenization_clip import CLIPTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_clip_fast import CLIPTokenizerFast

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_clip import CLIPFeatureExtractor
    from .image_processing_clip import CLIPImageProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_clip import (
        CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
        CLIPModel,
        CLIPPreTrainedModel,
        CLIPTextModel,
        CLIPTextModelWithProjection,
        CLIPVisionModel,
        CLIPVisionModelWithProjection,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_clip import (
        TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFCLIPModel,
        TFCLIPPreTrainedModel,
        TFCLIPTextModel,
        TFCLIPVisionModel,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_clip import (
        FlaxCLIPModel,
        FlaxCLIPPreTrainedModel,
        FlaxCLIPTextModel,
        FlaxCLIPTextPreTrainedModel,
        FlaxCLIPVisionModel,
        FlaxCLIPVisionPreTrainedModel,
    )


else:
  import sys

  __snake_case             =   _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)









"""simple docstring"""



import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
__snake_case             =   logging.getLogger(__name__)

if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser(
      description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
  )
  parser.add_argument(
      """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
  )
  parser.add_argument(
      """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
  )
  parser.add_argument("""--vocab_size""", default=30522, type=int)
  __snake_case             =   parser.parse_args()

  logger.info(F'''Loading data from {args.data_file}''')
  with open(args.data_file, """rb""") as fp:
    __snake_case             =   pickle.load(fp)

  logger.info("""Counting occurrences for MLM.""")
  __snake_case             =   Counter()
  for tk_ids in data:
    counter.update(tk_ids)
  __snake_case             =   [0] * args.vocab_size
  for k, v in counter.items():
    __snake_case             =   v

  logger.info(F'''Dump to {args.token_counts_dump}''')
  with open(args.token_counts_dump, """wb""") as handle:
    pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)





