





'''simple docstring'''

import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
  from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase							:     Tuple		     =get_tests_dir("fixtures/test_sentencepiece.model")

__lowerCAmelCase							:     List[str]		     ={"target_lang": "fi", "source_lang": "en"}
__lowerCAmelCase							:     Tuple		     =">>zh<<"
__lowerCAmelCase							:     str		     ="Helsinki-NLP/"

if is_torch_available():
  __lowerCAmelCase							:     List[str]		     ="pt"
elif is_tf_available():
  __lowerCAmelCase							:     Any		     ="tf"
else:
  __lowerCAmelCase							:     Optional[int]		     ="jax"






@require_sentencepiece
class        UpperCAmelCase     (	UpperCamelCase__      ,	unittest.TestCase    ):
 __lowercase       		=    MarianTokenizer
 __lowercase       		=    False
 __lowercase       		=    True
 def 					UpperCAmelCase_							(    self				:Union[str, Any]							)->							int:
     super().setUp()
     A__    =      ["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
     A__    =      dict(zip(lowercase_						,	range(len(lowercase_							)							)							)							)
     A__    =      Path(self.tmpdirname							)
     save_json(lowercase_						,	save_dir / VOCAB_FILES_NAMES["vocab"]							)
     save_json(lowercase_						,	save_dir / VOCAB_FILES_NAMES["tokenizer_config_file"]							)
     if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
         copyfile(lowercase_						,	save_dir / VOCAB_FILES_NAMES["source_spm"]							)
         copyfile(lowercase_						,	save_dir / VOCAB_FILES_NAMES["target_spm"]							)

     A__    =      MarianTokenizer.from_pretrained(self.tmpdirname							)
     tokenizer.save_pretrained(self.tmpdirname							)
 def 					UpperCAmelCase_							(    self				:int						,	**lowercase_				:Any							)->							MarianTokenizer:
     return MarianTokenizer.from_pretrained(self.tmpdirname						,	**lowercase_							)
 def 					UpperCAmelCase_							(    self				:Dict						,	lowercase_				:List[str]							)->							Optional[int]:
     return (
         "This is a test",
         "This is a test",
     )
 def 					UpperCAmelCase_							(    self				:List[str]							)->							Optional[int]:
     A__    =      "</s>"
     A__    =      0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_							)						,	lowercase_							)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_							)						,	lowercase_							)
 def 					UpperCAmelCase_							(    self				:Dict							)->							List[str]:
     A__    =      list(self.get_tokenizer().get_vocab().keys()							)

     self.assertEqual(vocab_keys[0]						,	"</s>"							)
     self.assertEqual(vocab_keys[1]						,	"<unk>"							)
     self.assertEqual(vocab_keys[-1]						,	"<pad>"							)
     self.assertEqual(len(lowercase_							)						,	9							)
 def 					UpperCAmelCase_							(    self				:Optional[Any]							)->							Optional[Any]:
     self.assertEqual(self.get_tokenizer().vocab_size						,	9							)
 def 					UpperCAmelCase_							(    self				:Dict							)->							int:
     A__    =      MarianTokenizer.from_pretrained(F"{ORG_NAME}opus-mt-en-de"							)
     A__    =      en_de_tokenizer(["I am a small frog"]						,	return_tensors=lowercase_							)
     self.assertIsInstance(lowercase_						,	lowercase_							)
     A__    =      [38, 1_21, 14, 6_97, 3_88_48, 0]
     self.assertListEqual(lowercase_						,	batch.input_ids[0]							)

     A__    =      tempfile.mkdtemp()
     en_de_tokenizer.save_pretrained(lowercase_							)
     A__    =      [x.name for x in Path(lowercase_							).glob("*"							)]
     self.assertIn("source.spm"						,	lowercase_							)
     MarianTokenizer.from_pretrained(lowercase_							)
 def 					UpperCAmelCase_							(    self				:Tuple							)->							Optional[int]:
     A__    =      self.get_tokenizer()

     A__    =      tok(
         ["I am a small frog" * 10_00, "I am a small frog"]						,	padding=lowercase_						,	truncation=lowercase_						,	return_tensors=lowercase_							)
     self.assertIsInstance(lowercase_						,	lowercase_							)
     self.assertEqual(batch.input_ids.shape						,	(2, 5_12)							)
 def 					UpperCAmelCase_							(    self				:Dict							)->							Union[str, Any]:
     A__    =      self.get_tokenizer()
     A__    =      tok(["I am a tiny frog", "I am a small frog"]						,	padding=lowercase_						,	return_tensors=lowercase_							)
     self.assertIsInstance(lowercase_						,	lowercase_							)
     self.assertEqual(batch_smaller.input_ids.shape						,	(2, 10)							)
 @slow
 def 					UpperCAmelCase_							(    self				:int							)->							Tuple:
     # fmt: off
     A__    =      {"input_ids": [[4_34_95, 4_62, 20, 4_21_64, 13_69, 52, 4_64, 1_32, 17_03, 4_92, 13, 74_91, 3_89_99, 6, 8, 4_64, 1_32, 17_03, 4_92, 13, 46_69, 3_78_67, 13, 75_25, 27, 15_93, 9_88, 13, 3_39_72, 70_29, 6, 20, 82_51, 3_83, 2, 2_70, 58_66, 37_88, 2, 23_53, 82_51, 1_23_38, 2, 1_39_58, 3_87, 2, 36_29, 69_53, 1_88, 29_00, 2, 1_39_58, 80_11, 1_15_01, 23, 84_60, 40_73, 3_40_09, 20, 4_35, 1_14_39, 27, 8, 84_60, 40_73, 60_04, 20, 99_88, 3_75, 27, 33, 2_66, 19_45, 10_76, 13_50, 3_78_67, 32_88, 5, 5_77, 10_76, 43_74, 8, 50_82, 5, 2_64_53, 2_57, 5_56, 4_03, 2, 2_42, 1_32, 3_83, 3_16, 4_92, 8, 1_07_67, 6, 3_16, 3_04, 42_39, 3, 0], [1_48, 1_57_22, 19, 18_39, 12, 13_50, 13, 2_23_27, 50_82, 54_18, 4_75_67, 3_59_38, 59, 3_18, 1_95_52, 1_08, 21_83, 54, 1_49_76, 48_35, 32, 5_47, 11_14, 8, 3_15, 24_17, 5, 92, 1_90_88, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00], [36, 63_95, 1_25_70, 3_91_47, 1_15_97, 6, 2_66, 4, 4_54_05, 72_96, 3, 0, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00, 5_81_00]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=lowercase_						,	model_name="Helsinki-NLP/opus-mt-en-de"						,	revision="1a8c2263da11e68e50938f97e10cd57820bd504c"						,	decode_kwargs={"use_source_tokenizer": True}						,	)



 def 					UpperCAmelCase_							(    self				:List[Any]							)->							Tuple:
     A__    =      MarianTokenizer.from_pretrained("hf-internal-testing/test-marian-two-vocabs"							)

     A__    =      "Tämä on testi"
     A__    =      "This is a test"

     A__    =      [76, 7, 20_47, 2]
     A__    =      [69, 12, 11, 9_40, 2]

     A__    =      tokenizer(lowercase_							).input_ids
     self.assertListEqual(lowercase_						,	lowercase_							)

     A__    =      tokenizer(text_target=lowercase_							).input_ids
     self.assertListEqual(lowercase_						,	lowercase_							)

     A__    =      tokenizer.decode(lowercase_						,	skip_special_tokens=lowercase_							)
     self.assertEqual(lowercase_						,	lowercase_							)










'''simple docstring'''

import argparse
import os
import re

import packaging.version


__lowerCAmelCase							:     List[Any]		     ="examples/"
__lowerCAmelCase							:     Dict		     ={
    "examples": (re.compile(R"^check_min_version\(\"[^\"]+\"\)\s*$", re.MULTILINE), "check_min_version(\"VERSION\")\n"),
    "init": (re.compile(R"^__version__\s+=\s+\"([^\"]+)\"\s*$", re.MULTILINE), "__version__ = \"VERSION\"\n"),
    "setup": (re.compile(R"^(\s*)version\s*=\s*\"[^\"]+\",", re.MULTILINE), R"\1version=\"VERSION\","),
    "doc": (re.compile(R"^(\s*)release\s*=\s*\"[^\"]+\"$", re.MULTILINE), "release = \"VERSION\"\n"),
}
__lowerCAmelCase							:     List[str]		     ={
    "init": "src/transformers/__init__.py",
    "setup": "setup.py",
}
__lowerCAmelCase							:     str		     ="README.md"





def 						UpperCamelCase    (							_lowerCamelCase : Optional[Any]						,	_lowerCamelCase : Union[str, Any]						,	_lowerCamelCase : Optional[Any]	):
    with open(_lowerCamelCase						,	"r"						,	encoding="utf-8"						,	newline="\n"	) as f:
        A__    =      f.read()
    A__,     A__    =      REPLACE_PATTERNS[pattern]
    A__    =      replace.replace("VERSION"						,	_lowerCamelCase	)
    A__    =      re_pattern.sub(_lowerCamelCase						,	_lowerCamelCase	)
    with open(_lowerCamelCase						,	"w"						,	encoding="utf-8"						,	newline="\n"	) as f:
        f.write(_lowerCamelCase	)





def 						UpperCamelCase    (							_lowerCamelCase : int	):
    for folder, directories, fnames in os.walk(_lowerCamelCase	):
        # Removing some of the folders with non-actively maintained examples from the walk
        if "research_projects" in directories:
            directories.remove("research_projects"	)
        if "legacy" in directories:
            directories.remove("legacy"	)
        for fname in fnames:
            if fname.endswith(".py"	):
                update_version_in_file(os.path.join(_lowerCamelCase						,	_lowerCamelCase	)						,	_lowerCamelCase						,	pattern="examples"	)





def 						UpperCamelCase    (							_lowerCamelCase : int						,	_lowerCamelCase : Union[str, Any]=False	):
    for pattern, fname in REPLACE_FILES.items():
        update_version_in_file(_lowerCamelCase						,	_lowerCamelCase						,	_lowerCamelCase	)
    if not patch:
        update_version_in_examples(_lowerCamelCase	)





def 						UpperCamelCase    (							):
    A__    =      "🤗 Transformers currently provides the following architectures"
    A__    =      "1. Want to contribute a new model?"
    with open(_lowerCamelCase						,	"r"						,	encoding="utf-8"						,	newline="\n"	) as f:
        A__    =      f.readlines()

    # Find the start of the list.
    A__    =      0
    while not lines[start_index].startswith(_start_prompt	):
        start_index += 1
    start_index += 1

    A__    =      start_index
    # Update the lines in the model list.
    while not lines[index].startswith(_end_prompt	):
        if lines[index].startswith("1."	):
            A__    =      lines[index].replace(
                "https://huggingface.co/docs/transformers/main/model_doc"						,	"https://huggingface.co/docs/transformers/model_doc"						,	)
        index += 1

    with open(_lowerCamelCase						,	"w"						,	encoding="utf-8"						,	newline="\n"	) as f:
        f.writelines(_lowerCamelCase	)





def 						UpperCamelCase    (							):
    with open(REPLACE_FILES["init"]						,	"r"	) as f:
        A__    =      f.read()
    A__    =      REPLACE_PATTERNS["init"][0].search(_lowerCamelCase	).groups()[0]
    return packaging.version.parse(_lowerCamelCase	)





def 						UpperCamelCase    (							_lowerCamelCase : Dict=False	):
    A__    =      get_version()
    if patch and default_version.is_devrelease:
        raise ValueError("Can't create a patch version from the dev branch, checkout a released version!"	)
    if default_version.is_devrelease:
        A__    =      default_version.base_version
    elif patch:
        A__    =      F"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
    else:
        A__    =      F"{default_version.major}.{default_version.minor + 1}.0"

    # Now let's ask nicely if that's the right one.
    A__    =      input(F"Which version are you releasing? [{default_version}]"	)
    if len(_lowerCamelCase	) == 0:
        A__    =      default_version

    print(F"Updating version to {version}."	)
    global_version_update(_lowerCamelCase						,	patch=_lowerCamelCase	)
    if not patch:
        print("Cleaning main README, don't forget to run `make fix-copies`."	)
        clean_main_ref_in_model_list()





def 						UpperCamelCase    (							):
    A__    =      get_version()
    A__    =      F"{current_version.major}.{current_version.minor + 1}.0.dev0"
    A__    =      current_version.base_version

    # Check with the user we got that right.
    A__    =      input(F"Which version are we developing now? [{dev_version}]"	)
    if len(_lowerCamelCase	) == 0:
        A__    =      dev_version

    print(F"Updating version to {version}."	)
    global_version_update(_lowerCamelCase	)
    print("Cleaning main README, don't forget to run `make fix-copies`."	)
    clean_main_ref_in_model_list()


if __name__ == "__main__":
  __lowerCAmelCase							:     Union[str, Any]		     =argparse.ArgumentParser()
  parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.")
  parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.")
  __lowerCAmelCase							:     int		     =parser.parse_args()
  if not args.post_release:
    pre_release_work(patch=args.patch)
  elif args.patch:
    print("Nothing to do after a patch :-)")
  else:
    post_release_work()



