





'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


__lowerCAmelCase							:     str		     =logging.get_logger(__name__)


__lowerCAmelCase							:     str		     ={"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all LED models at https://huggingface.co/models?filter=LED
__lowerCAmelCase							:     Dict		     ={
    "vocab_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json",
    },
    "merges_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json",
    },
}

__lowerCAmelCase							:     Optional[Any]		     ={
    "allenai/led-base-16384": 1_6384,
}





@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 						UpperCamelCase    (							):
   A__    =      (
       list(range(ord("!"	)						,	ord("~"	) + 1	)	) + list(range(ord("¡"	)						,	ord("¬"	) + 1	)	) + list(range(ord("®"	)						,	ord("ÿ"	) + 1	)	)
   )
   A__    =      bs[:]
   A__    =      0
   for b in range(2**8	):
      if b not in bs:
         bs.append(_lowerCamelCase	)
         cs.append(2**8 + n	)
         n += 1
   A__    =      [chr(_lowerCamelCase	) for n in cs]
   return dict(zip(_lowerCamelCase						,	_lowerCamelCase	)	)





def 						UpperCamelCase    (							_lowerCamelCase : Dict	):
   A__    =      set()
   A__    =      word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)	)
      A__    =      char
   return pairs






class        UpperCAmelCase     (	UpperCamelCase__    ):
 __lowercase       		=    VOCAB_FILES_NAMES
 __lowercase       		=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase       		=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase       		=    ["""input_ids""", """attention_mask"""]
 def __init__(    self				:Any						,	lowercase_				:Optional[Any]						,	lowercase_				:Any						,	lowercase_				:List[str]="replace"						,	lowercase_				:str="<s>"						,	lowercase_				:Union[str, Any]="</s>"						,	lowercase_				:str="</s>"						,	lowercase_				:Optional[Any]="<s>"						,	lowercase_				:Dict="<unk>"						,	lowercase_				:Tuple="<pad>"						,	lowercase_				:Optional[int]="<mask>"						,	lowercase_				:Any=False						,	**lowercase_				:Dict						,	)->							Optional[Any]:
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else bos_token
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else eos_token
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else sep_token
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else cls_token
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else unk_token
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    A__    =      AddedToken(lowercase_						,	lstrip=lowercase_						,	rstrip=lowercase_							) if isinstance(lowercase_						,	lowercase_							) else mask_token

    super().__init__(
        errors=lowercase_						,	bos_token=lowercase_						,	eos_token=lowercase_						,	unk_token=lowercase_						,	sep_token=lowercase_						,	cls_token=lowercase_						,	pad_token=lowercase_						,	mask_token=lowercase_						,	add_prefix_space=lowercase_						,	**lowercase_						,	)

    with open(lowercase_						,	encoding="utf-8"							) as vocab_handle:
       A__    =      json.load(lowercase_							)
    A__    =      {v: k for k, v in self.encoder.items()}
    A__    =      errors  # how to handle errors in decoding
    A__    =      bytes_to_unicode()
    A__    =      {v: k for k, v in self.byte_encoder.items()}
    with open(lowercase_						,	encoding="utf-8"							) as merges_handle:
       A__    =      merges_handle.read().split("\n"							)[1:-1]
    A__    =      [tuple(merge.split()							) for merge in bpe_merges]
    A__    =      dict(zip(lowercase_						,	range(len(lowercase_							)							)							)							)
    A__    =      {}
    A__    =      add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    A__    =      re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"							)
 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def 					UpperCAmelCase_							(    self				:str							)->							Union[str, Any]:
    return len(self.encoder							)
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
    return dict(self.encoder						,	**self.added_tokens_encoder							)
 def 					UpperCAmelCase_							(    self				:Union[str, Any]						,	lowercase_				:List[Any]							)->							Dict:
    if token in self.cache:
       return self.cache[token]
    A__    =      tuple(lowercase_							)
    A__    =      get_pairs(lowercase_							)

    if not pairs:
       return token

    while True:
       A__    =      min(lowercase_						,	key=lambda lowercase_							: self.bpe_ranks.get(lowercase_						,	float("inf"							)							)							)
       if bigram not in self.bpe_ranks:
          break
       A__,     A__    =      bigram
       A__    =      []
       A__    =      0
       while i < len(lowercase_							):
          try:
             A__    =      word.index(lowercase_						,	lowercase_							)
          except ValueError:
             new_word.extend(word[i:]							)
             break
          else:
             new_word.extend(word[i:j]							)
             A__    =      j

          if word[i] == first and i < len(lowercase_							) - 1 and word[i + 1] == second:
             new_word.append(first + second							)
             i += 2
          else:
             new_word.append(word[i]							)
             i += 1
       A__    =      tuple(lowercase_							)
       A__    =      new_word
       if len(lowercase_							) == 1:
          break
       else:
          A__    =      get_pairs(lowercase_							)
    A__    =      " ".join(lowercase_							)
    A__    =      word
    return word
 def 					UpperCAmelCase_							(    self				:List[Any]						,	lowercase_				:Any							)->							Any:
    A__    =      []
    for token in re.findall(self.pat						,	lowercase_							):
       A__    =      "".join(
           self.byte_encoder[b] for b in token.encode("utf-8"							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_							).split(" "							)							)
    return bpe_tokens
 def 					UpperCAmelCase_							(    self				:Optional[Any]						,	lowercase_				:List[str]							)->							str:
    return self.encoder.get(lowercase_						,	self.encoder.get(self.unk_token							)							)
 def 					UpperCAmelCase_							(    self				:Optional[Any]						,	lowercase_				:Tuple							)->							str:
    return self.decoder.get(lowercase_							)
 def 					UpperCAmelCase_							(    self				:Dict						,	lowercase_				:Dict							)->							int:
    A__    =      "".join(lowercase_							)
    A__    =      bytearray([self.byte_decoder[c] for c in text]							).decode("utf-8"						,	errors=self.errors							)
    return text
 def 					UpperCAmelCase_							(    self				:int						,	lowercase_				:str						,	lowercase_				:Optional[str] = None							)->							Tuple[str]:
    if not os.path.isdir(lowercase_							):
       logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
       return
    A__    =      os.path.join(
        lowercase_						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)
    A__    =      os.path.join(
        lowercase_						,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]							)

    with open(lowercase_						,	"w"						,	encoding="utf-8"							) as f:
       f.write(json.dumps(self.encoder						,	indent=2						,	sort_keys=lowercase_						,	ensure_ascii=lowercase_							) + "\n"							)

    A__    =      0
    with open(lowercase_						,	"w"						,	encoding="utf-8"							) as writer:
       writer.write("#version: 0.2\n"							)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,	key=lambda lowercase_							: kv[1]							):
          if index != token_index:
             logger.warning(
                 F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                 " Please check that the tokenizer is not corrupted!"							)
             A__    =      token_index
          writer.write(" ".join(lowercase_							) + "\n"							)
          index += 1

    return vocab_file, merge_file
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	lowercase_				:List[int]						,	lowercase_				:Optional[List[int]] = None							)->							List[int]:
    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    A__    =      [self.cls_token_id]
    A__    =      [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep
 def 					UpperCAmelCase_							(    self				:Union[str, Any]						,	lowercase_				:List[int]						,	lowercase_				:Optional[List[int]] = None						,	lowercase_				:bool = False							)->							List[int]:
    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=lowercase_						,	token_ids_a=lowercase_						,	already_has_special_tokens=lowercase_							)

    if token_ids_a is None:
       return [1] + ([0] * len(lowercase_							)) + [1]
    return [1] + ([0] * len(lowercase_							)) + [1, 1] + ([0] * len(lowercase_							)) + [1]
 def 					UpperCAmelCase_							(    self				:int						,	lowercase_				:List[int]						,	lowercase_				:Optional[List[int]] = None							)->							List[int]:
    A__    =      [self.sep_token_id]
    A__    =      [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep							) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]
 def 					UpperCAmelCase_							(    self				:str						,	lowercase_				:Union[str, Any]						,	lowercase_				:Union[str, Any]=False						,	**lowercase_				:Optional[int]							)->							List[str]:
    A__    =      kwargs.pop("add_prefix_space"						,	self.add_prefix_space							)
    if (is_split_into_words or add_prefix_space) and (len(lowercase_							) > 0 and not text[0].isspace()):
       A__    =      " " + text
    return (text, kwargs)



 def 					UpperCAmelCase_							(    self				:str						,	lowercase_				:Union[Dict[str, EncodedInput], BatchEncoding]						,	lowercase_				:Optional[int] = None						,	lowercase_				:PaddingStrategy = PaddingStrategy.DO_NOT_PAD						,	lowercase_				:Optional[int] = None						,	lowercase_				:Optional[bool] = None						,	)->							dict:
    A__    =      super()._pad(
        encoded_inputs=lowercase_						,	max_length=lowercase_						,	padding_strategy=lowercase_						,	pad_to_multiple_of=lowercase_						,	return_attention_mask=lowercase_						,	)

    # Load from model defaults
    if return_attention_mask is None:
       A__    =      "attention_mask" in self.model_input_names

    if return_attention_mask and "global_attention_mask" in encoded_inputs:
       A__    =      encoded_inputs[self.model_input_names[0]]
       # `global_attention_mask` need to have the same length as other (sequential) inputs.
       A__    =      len(encoded_inputs["global_attention_mask"]							) != len(lowercase_							)

       if needs_to_be_padded:
          A__    =      len(lowercase_							) - len(encoded_inputs["global_attention_mask"]							)

          if self.padding_side == "right":
             # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
             A__    =      (
                 encoded_inputs["global_attention_mask"] + [-1] * difference
             )
          elif self.padding_side == "left":
             A__    =      [-1] * difference + encoded_inputs[
                 "global_attention_mask"
             ]
          else:
             raise ValueError("Invalid padding strategy:" + str(self.padding_side							)							)

    return encoded_inputs










'''simple docstring'''

import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class        UpperCAmelCase     (	UpperCamelCase__      ,	unittest.TestCase    ):
 __lowercase       		=    CLIPTokenizer
 __lowercase       		=    CLIPTokenizerFast
 __lowercase       		=    True
 __lowercase       		=    {}
 __lowercase       		=    False
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     super().setUp()

     # fmt: off
     A__    =      ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
     # fmt: on
     A__    =      dict(zip(lowercase_						,	range(len(lowercase_							)							)							)							)
     A__    =      ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
     A__    =      {"unk_token": "<unk>"}

     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]							)
     A__    =      os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]							)
     with open(self.vocab_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write(json.dumps(lowercase_							) + "\n"							)
     with open(self.merges_file						,	"w"						,	encoding="utf-8"							) as fp:
         fp.write("\n".join(lowercase_							)							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:List[str]							)->							str:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizer.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	**lowercase_				:Optional[int]							)->							Tuple:
     kwargs.update(self.special_tokens_map							)
     return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]						,	lowercase_				:List[Any]							)->							List[str]:
     A__    =      "lower newer"
     A__    =      "lower newer"
     return input_text, output_text
 def 					UpperCAmelCase_							(    self				:int							)->							List[str]:
     A__    =      CLIPTokenizer(self.vocab_file						,	self.merges_file						,	**self.special_tokens_map							)
     A__    =      "lower newer"
     A__    =      ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
     A__    =      tokenizer.tokenize(lowercase_							)
     self.assertListEqual(lowercase_						,	lowercase_							)

     A__    =      tokens + [tokenizer.unk_token]
     A__    =      [10, 2, 16, 9, 3, 2, 16, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_							)						,	lowercase_							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:Dict							)->							str:
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      self.tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)
             A__    =      self.rust_tokenizer_class.from_pretrained(lowercase_						,	**lowercase_							)

             A__    =      "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
             # with Tilde) encoded in 2 different ways
             A__    =      "xa\u0303y" + " " + "x\xe3y"
             A__    =      tokenizer_s.tokenize(lowercase_							)
             A__    =      tokenizer_r.tokenize(lowercase_							)

             self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of space type
             A__    =      [
                 "\u0009",  # (horizontal tab, '\t')
                 "\u000B",  # (vertical tab)
                 "\u000C",  # (form feed)
                 "\u0020",  # (space, ' ')
                 "\u200E",  # (left-to-right mark):w
                 "\u200F",  # (right-to-left mark)
             ]
             for unicode_seq in spaces_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)

             # Test that the tokenization is identical on unicode of line break type
             A__    =      [
                 "\u000A",  # (line feed, '\n')
                 "\r\n",  # (carriage return and line feed, '\r\n')
                 "\u000D",  # (carriage return, '\r')
                 "\r",  # (carriage return, '\r')
                 "\u000D",  # (carriage return, '\r')
                 "\u2028",  # (line separator)
                 "\u2029",  # (paragraph separator)
                 # "\u0085", # (next line)
             ]

             # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
             # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
             # space (and thus into an empty list).

             for unicode_seq in line_break_unicodes:
                 A__    =      tokenizer_s.tokenize(lowercase_							)
                 A__    =      tokenizer_r.tokenize(lowercase_							)

                 self.assertListEqual(lowercase_						,	lowercase_							)
 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Optional[Any]:
     # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
             A__    =      "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
             A__    =      F"{text_of_1_token} {text_of_1_token}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(0, len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(len(lowercase_							) + 1, len(lowercase_							) + 1 + len(lowercase_							))						,	)

             A__    =      F" {text}"

             A__    =      self.rust_tokenizer_class.from_pretrained(
                 lowercase_						,	use_fast=lowercase_						,	)
             A__    =      tokenizer_r(lowercase_						,	return_offsets_mapping=lowercase_						,	add_special_tokens=lowercase_							)
             self.assertEqual(encoding.offset_mapping[0]						,	(1, 1 + len(lowercase_							))							)
             self.assertEqual(
                 encoding.offset_mapping[1]						,	(1 + len(lowercase_							) + 1, 1 + len(lowercase_							) + 1 + len(lowercase_							))						,	)
 def 					UpperCAmelCase_							(    self				:Dict							)->							Optional[Any]:
     # Test related to the breaking change introduced in transformers v4.17.0
     # We need to check that an error in raised when the user try to load a previous version of the tokenizer.
     with self.assertRaises(lowercase_							) as context:
         self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"							)

     self.assertTrue(
         context.exception.args[0].startswith(
             "The `backend_tokenizer` provided does not match the expected format."							)							)
 @require_ftfy
 def 					UpperCAmelCase_							(    self				:str							)->							Any:
     super().test_tokenization_python_rust_equals()



 def 					UpperCAmelCase_							(    self				:Optional[int]							)->							Union[str, Any]:
     # CLIP always lower cases letters
     pass



