





# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path



def 		A_    (					_lowerCAmelCase							,       _lowerCAmelCase							,       _lowerCAmelCase							,       _lowerCAmelCase					)						->			Union[str, Any]:
 UpperCamelCase     : Dict	     =       {
     "en": "Machine learning is great, isn't it?",
     "ru": "Машинное обучение - это здорово, не так ли?",
     "de": "Maschinelles Lernen ist großartig, nicht wahr?",
 }

 # BLUE scores as follows:
 # "pair": [fairseq, transformers]
 UpperCamelCase     : Tuple	     =       {
     "wmt16-en-de-dist-12-1": [28.3, 27.52],
     "wmt16-en-de-dist-6-1": [27.4, 27.11],
     "wmt16-en-de-12-1": [26.9, 25.75],
 }
 UpperCamelCase     : int	     =       F"""{src_lang}-{tgt_lang}"""

 UpperCamelCase     : List[str]	     =       F"""
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"allenai/{model_name}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

"""
 model_card_dir.mkdir(parents=_lowerCAmelCase							,       exist_ok=_lowerCAmelCase					)
 UpperCamelCase     : Any	     =       os.path.join(_lowerCAmelCase							,       "README.md"					)
 print(F"""Generating {path}"""					)
 with open(_lowerCAmelCase							,       "w"							,       encoding="utf-8"					) as f:
  f.write(_lowerCAmelCase					)


# make sure we are under the root of the project
__lowerCamelCase	:					List[Any]			  =					Path(__file__).resolve().parent.parent.parent
__lowerCamelCase	:					List[str]			  =					repo_dir / """model_cards"""

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
    __lowerCamelCase	:					int			  =					model_cards_dir / """allenai""" / model_name
    write_model_card(model_card_dir, src_lang="""en""", tgt_lang="""de""", model_name=model_name)







from typing import Callable, Dict, Optional, Tuple

import torch
from torch import nn
from torch.distributions import (
    AffineTransform,
    Distribution,
    Independent,
    NegativeBinomial,
    Normal,
    StudentT,
    TransformedDistribution,
)



class  A__       (	__snake_case      ):





      def __init__(			self	,							A_	,							A_=None	,							A_=None	,							A_=0       ):
       '''simple docstring'''




       UpperCamelCase     : Union[str, Any]	     =       1.0 if scale is None else scale
       UpperCamelCase     : Optional[int]	     =       0.0 if loc is None else loc

       super().__init__(A_	,							[AffineTransform(loc=self.loc	,							scale=self.scale	,							event_dim=A_       )]       )





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return self.base_dist.mean * self.scale + self.loc





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return self.base_dist.variance * self.scale**2





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return self.variance.sqrt()



class  A__       (	nn.Module      ):





      def __init__(			self	,							A_	,							A_	,							A_	,							**A_       ):
       '''simple docstring'''




       super().__init__(**A_       )
       UpperCamelCase     : Union[str, Any]	     =       args_dim
       UpperCamelCase     : str	     =       nn.ModuleList([nn.Linear(A_	,							A_       ) for dim in args_dim.values()]       )
       UpperCamelCase     : Union[str, Any]	     =       domain_map





      def 	__UpperCamelCase(			self	,							A_       ):
       '''simple docstring'''




       UpperCamelCase     : List[Any]	     =       [proj(A_       ) for proj in self.proj]

       return self.domain_map(*A_       )



class  A__       (	nn.Module      ):





      def __init__(			self	,							A_       ):
       '''simple docstring'''




       super().__init__()
       UpperCamelCase     : str	     =       function





      def 	__UpperCamelCase(			self	,							A_	,							*A_       ):
       '''simple docstring'''




       return self.function(A_	,							*A_       )



class  A__       :
      _UpperCAmelCase     :type
      _UpperCAmelCase     :int
      _UpperCAmelCase     :Dict[str, int]





      def __init__(			self	,							A_ = 1       ):
       '''simple docstring'''




       UpperCamelCase     : Tuple	     =       dim
       UpperCamelCase     : Union[str, Any]	     =       {k: dim * self.args_dim[k] for k in self.args_dim}





      def 	__UpperCamelCase(			self	,							A_       ):
       '''simple docstring'''




       if self.dim == 1:
        return self.distribution_class(*A_       )
       else:
        return Independent(self.distribution_class(*A_       )	,							1       )





      def 	__UpperCamelCase(			self	,							A_	,							A_ = None	,							A_ = None	,							):
       '''simple docstring'''




       UpperCamelCase     : str	     =       self._base_distribution(A_       )
       if loc is None and scale is None:
        return distr
       else:
        return AffineTransformed(A_	,							loc=A_	,							scale=A_	,							event_dim=self.event_dim       )





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return () if self.dim == 1 else (self.dim,)





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return len(self.event_shape       )





      @property
      def 	__UpperCamelCase(			self       ):
       '''simple docstring'''




       return 0.0





      def 	__UpperCamelCase(			self	,							A_       ):
       '''simple docstring'''




       return ParameterProjection(
           in_features=A_	,							args_dim=self.args_dim	,							domain_map=LambdaLayer(self.domain_map       )	,							)





      def 	__UpperCamelCase(			self	,							*A_       ):
       '''simple docstring'''




       raise NotImplementedError()





      @staticmethod
      def 	__UpperCamelCase(			A_       ):
       '''simple docstring'''




       return (x + torch.sqrt(torch.square(A_       ) + 4.0       )) / 2.0



class  A__       (	__snake_case      ):
      _UpperCAmelCase     :Dict[str, int]						   =       {"df": 1, "loc": 1, "scale": 1}
      _UpperCAmelCase     :type						   =       StudentT





      @classmethod
      def 	__UpperCamelCase(			cls	,							A_	,							A_	,							A_       ):
       '''simple docstring'''




       UpperCamelCase     : Optional[Any]	     =       cls.squareplus(A_       ).clamp_min(torch.finfo(scale.dtype       ).eps       )
       UpperCamelCase     : int	     =       2.0 + cls.squareplus(A_       )
       return df.squeeze(-1       ), loc.squeeze(-1       ), scale.squeeze(-1       )



class  A__       (	__snake_case      ):
      _UpperCAmelCase     :Dict[str, int]						   =       {"loc": 1, "scale": 1}
      _UpperCAmelCase     :type						   =       Normal





      @classmethod
      def 	__UpperCamelCase(			cls	,							A_	,							A_       ):
       '''simple docstring'''




       UpperCamelCase     : Dict	     =       cls.squareplus(A_       ).clamp_min(torch.finfo(scale.dtype       ).eps       )
       return loc.squeeze(-1       ), scale.squeeze(-1       )



class  A__       (	__snake_case      ):
      _UpperCAmelCase     :Dict[str, int]						   =       {"total_count": 1, "logits": 1}
      _UpperCAmelCase     :type						   =       NegativeBinomial





      @classmethod
      def 	__UpperCamelCase(			cls	,							A_	,							A_       ):
       '''simple docstring'''




       UpperCamelCase     : List[str]	     =       cls.squareplus(A_       )
       return total_count.squeeze(-1       ), logits.squeeze(-1       )





      def 	__UpperCamelCase(			self	,							A_       ):
       '''simple docstring'''




       UpperCamelCase     ,   UpperCamelCase     : Optional[int]	     =       distr_args
       if self.dim == 1:
        return self.distribution_class(total_count=A_	,							logits=A_       )
       else:
        return Independent(self.distribution_class(total_count=A_	,							logits=A_       )	,							1       )





      def 	__UpperCamelCase(			self	,							A_	,							A_ = None	,							A_ = None       ):
       '''simple docstring'''




       UpperCamelCase     ,   UpperCamelCase     : Any	     =       distr_args

       if scale is not None:
        # See scaling property of Gamma.
        logits += scale.log()

       return self._base_distribution((total_count, logits)       )
