Abstractive: Always create tokenizer (Fixes #44)

HHousen · Feb 9, 2021 · 9294897 · 9294897
1 parent 28f1fc2
commit 9294897
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/src/abstractive.py b/src/abstractive.py
@@ -112,9 +112,9 @@ def __init__(self, hparams):
                 gradient_checkpointing=self.hparams.gradient_checkpointing,
             )
 
-            self.tokenizer = AutoTokenizer.from_pretrained(
-                self.hparams.model_name_or_path, use_fast=True
-            )
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.hparams.model_name_or_path, use_fast=True
+        )
 
         if self.hparams.model_max_length:
             self.tokenizer.model_max_length = self.hparams.model_max_length