Upload tokenization_plasmid_lm.py with huggingface_hub

tokenization_plasmid_lm.py

@@ -56,28 +56,15 @@ class PlasmidLMTokenizer(PreTrainedTokenizer):
             if tok in self._vocab:
                 special_kwargs[name] = tok
 
+        # Disable automatic BOS/EOS insertion (transformers 5.x); prompts
+        # already contain <BOS>...<SEP> and the model generates <EOS> itself.
+        kwargs.setdefault("special_tokens_pattern", "none")
         super().__init__(**special_kwargs, **kwargs)
 
     @property
     def vocab_size(self) -> int:
         return len(self._vocab)
 
-    @property
-    def pad_token_id(self) -> int:
-        return self._vocab.get("<PAD>", 0)
-
-    @property
-    def bos_token_id(self) -> int:
-        return self._vocab.get("<BOS>", 1)
-
-    @property
-    def eos_token_id(self) -> int:
-        return self._vocab.get("<EOS>", 2)
-
-    @property
-    def sep_token_id(self) -> int:
-        return self._vocab.get("<SEP>", 3)
-
     def get_vocab(self) -> dict:
         return dict(self._vocab)