



import absl  # noqa: F401 # Here to have a nice missing dependency error message early on
import nltk  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import six  # noqa: F401 # Here to have a nice missing dependency error message early on
from rouge_score import rouge_scorer, scoring

import datasets


__UpperCAmelCase	       =	'''\
@inproceedings{lin-2004-rouge,
    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
    author = "Lin, Chin-Yew",
    booktitle = "Text Summarization Branches Out",
    month = jul,
    year = "2004",
    address = "Barcelona, Spain",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/W04-1013",
    pages = "74--81",
}
'''

__UpperCAmelCase	       =	'''\
ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
evaluating automatic summarization and machine translation software in natural language processing.
The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.

Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.

This metrics is a wrapper around Google Research reimplementation of ROUGE:
https://github.com/google-research/google-research/tree/master/rouge
'''

__UpperCAmelCase	       =	'''
Calculates average rouge scores for a list of hypotheses and references
Args:
    predictions: list of predictions to score. Each prediction
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
    rouge_types: A list of rouge types to calculate.
        Valid names:
        `"rouge{n}"` (e.g. `"rouge1"`, `"rouge2"`) where: {n} is the n-gram based scoring,
        `"rougeL"`: Longest common subsequence based scoring.
        `"rougeLSum"`: rougeLsum splits text using `"\n"`.
        See details in https://github.com/huggingface/datasets/issues/617
    use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes.
    use_aggregator: Return aggregates if this is set to True
Returns:
    rouge1: rouge_1 (precision, recall, f1),
    rouge2: rouge_2 (precision, recall, f1),
    rougeL: rouge_l (precision, recall, f1),
    rougeLsum: rouge_lsum (precision, recall, f1)
Examples:

    >>> rouge = datasets.load_metric(\'rouge\')
    >>> predictions = ["hello there", "general kenobi"]
    >>> references = ["hello there", "general kenobi"]
    >>> results = rouge.compute(predictions=predictions, references=references)
    >>> print(list(results.keys()))
    [\'rouge1\', \'rouge2\', \'rougeL\', \'rougeLsum\']
    >>> print(results["rouge1"])
    AggregateScore(low=Score(precision=1.0, recall=1.0, fmeasure=1.0), mid=Score(precision=1.0, recall=1.0, fmeasure=1.0), high=Score(precision=1.0, recall=1.0, fmeasure=1.0))
    >>> print(results["rouge1"].mid.fmeasure)
    1.0
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,   _KWARGS_DESCRIPTION							)
class     lowerCAmelCase_		(  datasets.Metric							):
       def  snake_case_	(      self	)					->		List[str]:
          return datasets.MetricInfo(
              description=_DESCRIPTION,      citation=_CITATION,      inputs_description=_KWARGS_DESCRIPTION,      features=datasets.Features(
                  {
                      'predictions': datasets.Value('string',      id='sequence'	),
                      'references': datasets.Value('string',      id='sequence'	),
                  }	),      codebase_urls=['https://github.com/google-research/google-research/tree/master/rouge'],      reference_urls=[
                  'https://en.wikipedia.org/wiki/ROUGE_(metric)',
                  'https://github.com/google-research/google-research/tree/master/rouge',
              ],      )






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=False	)					->		str:
          if rouge_types is None:
             UpperCamelCase   :  Optional[int]						     =     ['rouge1', 'rouge2', 'rougeL', 'rougeLsum']

          UpperCamelCase   :  Optional[int]						     =     rouge_scorer.RougeScorer(rouge_types=SCREAMING_SNAKE_CASE_,      use_stemmer=SCREAMING_SNAKE_CASE_	)
          if use_aggregator:
             UpperCamelCase   :  Optional[int]						     =     scoring.BootstrapAggregator()
          else:
             UpperCamelCase   :  List[Any]						     =     []

          for ref, pred in zip(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	):
             UpperCamelCase   :  List[Any]						     =     scorer.score(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)
             if use_aggregator:
                aggregator.add_scores(SCREAMING_SNAKE_CASE_	)
             else:
                scores.append(SCREAMING_SNAKE_CASE_	)

          if use_aggregator:
             UpperCamelCase   :  List[Any]						     =     aggregator.aggregate()
          else:
             UpperCamelCase   :  Dict						     =     {}
             for key in scores[0]:
                UpperCamelCase   :  Tuple						     =     [score[key] for score in scores]

          return result








import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__UpperCAmelCase	       =	{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}



@is_pipeline_test
class     lowerCAmelCase_		(  unittest.TestCase							):
       UpperCAmelCase__    : Dict       					=  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
       UpperCAmelCase__    : List[str]       					=  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

       if model_mapping is not None:
              UpperCAmelCase__    : Optional[Any]       					=  {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
       if tf_model_mapping is not None:
              UpperCAmelCase__    : Dict       					=  {
                  config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
              }
       @require_torch
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  List[str]						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='pt'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          UpperCamelCase   :  Dict						     =     text_classifier('This is great !',      top_k=2	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]	)

          UpperCamelCase   :  List[Any]						     =     text_classifier(['This is great !', 'This is bad'],      top_k=2	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
              ],      )

          UpperCamelCase   :  List[Any]						     =     text_classifier('This is great !',      top_k=1	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          # Legacy behavior
          UpperCamelCase   :  str						     =     text_classifier('This is great !',      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)

          UpperCamelCase   :  Tuple						     =     text_classifier('This is great !',      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [[{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}]]	)

          UpperCamelCase   :  List[Any]						     =     text_classifier(['This is great !', 'Something else'],      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
                  [{'label': 'LABEL_0', 'score': 0.5_04}, {'label': 'LABEL_1', 'score': 0.4_96}],
              ],      )

          UpperCamelCase   :  Optional[int]						     =     text_classifier(['This is great !', 'Something else'],      return_all_scores=SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [
                  {'label': 'LABEL_0', 'score': 0.5_04},
                  {'label': 'LABEL_0', 'score': 0.5_04},
              ],      )
       @require_torch
       def  snake_case_	(      self	)					->		Optional[Any]:
          import torch

          UpperCamelCase   :  str						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='pt',      device=torch.device('cpu'	),      )

          UpperCamelCase   :  List[Any]						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)
       @require_tf
       def  snake_case_	(      self	)					->		Dict:
          UpperCamelCase   :  List[Any]						     =     pipeline(
              task='text-classification',      model='hf-internal-testing/tiny-random-distilbert',      framework='tf'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'LABEL_0', 'score': 0.5_04}]	)
       @slow
       @require_torch
       def  snake_case_	(      self	)					->		int:
          UpperCamelCase   :  str						     =     pipeline('text-classification'	)

          UpperCamelCase   :  str						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 1.0}]	)
          UpperCamelCase   :  List[str]						     =     text_classifier('This is bad !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'NEGATIVE', 'score': 1.0}]	)
          UpperCamelCase   :  int						     =     text_classifier('Birds are a type of animal'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 0.9_88}]	)
       @slow
       @require_tf
       def  snake_case_	(      self	)					->		Tuple:
          UpperCamelCase   :  int						     =     pipeline('text-classification',      framework='tf'	)

          UpperCamelCase   :  Tuple						     =     text_classifier('This is great !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 1.0}]	)
          UpperCamelCase   :  Union[str, Any]						     =     text_classifier('This is bad !'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'NEGATIVE', 'score': 1.0}]	)
          UpperCamelCase   :  int						     =     text_classifier('Birds are a type of animal'	)
          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': 'POSITIVE', 'score': 0.9_88}]	)
       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)					->		Union[str, Any]:
          UpperCamelCase   :  str						     =     TextClassificationPipeline(model=SCREAMING_SNAKE_CASE_,      tokenizer=SCREAMING_SNAKE_CASE_	)
          return text_classifier, ["HuggingFace is in", "This is another test"]






       def  snake_case_	(      self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_	)					->		Any:
          UpperCamelCase   :  List[str]						     =     text_classifier.model
          # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
          UpperCamelCase   :  List[str]						     =     'HuggingFace is in'
          UpperCamelCase   :  Tuple						     =     text_classifier(SCREAMING_SNAKE_CASE_	)

          self.assertEqual(nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}]	)
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)

          UpperCamelCase   :  List[Any]						     =     ['HuggingFace is in ', 'Paris is in France']
          UpperCamelCase   :  List[str]						     =     text_classifier(SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}, {'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}],      )
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)
          self.assertTrue(outputs[1]['label'] in model.config.idalabel.values()	)

          # Forcing to get all results with `top_k=None`
          # This is NOT the legacy format
          UpperCamelCase   :  Optional[int]						     =     text_classifier(SCREAMING_SNAKE_CASE_,      top_k=SCREAMING_SNAKE_CASE_	)
          UpperCamelCase   :  str						     =     len(model.config.idalabel.values()	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [[{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}] * N, [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}] * N],      )

          UpperCamelCase   :  str						     =     {'text': 'HuggingFace is in ', 'text_pair': 'Paris is in France'}
          UpperCamelCase   :  Union[str, Any]						     =     text_classifier(SCREAMING_SNAKE_CASE_	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      {'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)},      )
          self.assertTrue(outputs['label'] in model.config.idalabel.values()	)

          # This might be used a text pair, but tokenizer + pipe interaction
          # makes it hard to understand that it's not using the pair properly
          # https://github.com/huggingface/transformers/issues/17305
          # We disabled this usage instead as it was outputting wrong outputs.
          UpperCamelCase   :  List[str]						     =     [['HuggingFace is in ', 'Paris is in France']]
          with self.assertRaises(SCREAMING_SNAKE_CASE_	):
             text_classifier(SCREAMING_SNAKE_CASE_	)

          # This used to be valid for doing text pairs
          # We're keeping it working because of backward compatibility
          UpperCamelCase   :  List[str]						     =     text_classifier([[['HuggingFace is in ', 'Paris is in France']]]	)
          self.assertEqual(
              nested_simplify(SCREAMING_SNAKE_CASE_	),      [{'label': ANY(SCREAMING_SNAKE_CASE_	), 'score': ANY(SCREAMING_SNAKE_CASE_	)}],      )
          self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()	)



