Rename 'is_pretokenized' to 'is_split_into_words' for Tokenizers

HHousen · May 31, 2021 · 03eeee8 · 03eeee8
1 parent 20c8cac
commit 03eeee8
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/abstractive.py b/src/abstractive.py
@@ -385,7 +385,7 @@ def convert_to_features(example_batch):
                 sents_input_ids = self.tokenizer.encode_plus(
                     sents_tokenized_flat,
                     truncation=True,
-                    is_pretokenized=True,
+                    is_split_into_words=True,
                     add_special_tokens=False,
                     max_length=(max_length - 2),
                     return_attention_mask=False,