Do not break compilation for users with old SentencePiece versions (#40)

OpenNMT · Oct 1, 2018 · 5eb614a · 5eb614a
1 parent e1a0a93
commit 5eb614a
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 ### Fixes and improvements
 
+* Do not break compilation for users with old SentencePiece versions
+
 ## [v1.9.0](https://github.com/OpenNMT/Tokenizer/releases/tag/v1.9.0) (2018-09-25)
 
 ### New features

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -83,6 +83,11 @@ else()
   if(HAS_SAMPLE_ENCODE)
     add_definitions(-DSP_HAS_SAMPLE_ENCODE)
   endif()
+
+  file(STRINGS ${SP_INCLUDE_DIR}/sentencepiece_processor.h HAS_VOCAB_RESTRICTION REGEX "SetVocabulary")
+  if(HAS_VOCAB_RESTRICTION)
+    add_definitions(-DSP_HAS_VOCAB_RESTRICTION)
+  endif()
 endif()
 
 add_library(${PROJECT_NAME} ${SOURCES})

diff --git a/src/SentencePiece.cc b/src/SentencePiece.cc
@@ -21,12 +21,22 @@ namespace onmt
 
   void SentencePiece::set_vocabulary(const std::vector<std::string>& vocabulary)
   {
+#ifdef SP_HAS_VOCAB_RESTRICTION
     _processor.SetVocabulary(vocabulary);
+#else
+    throw std::runtime_error("The project was built against a SentencePiece version "
+                             "that does not support vocabulary restriction");
+#endif
   }
 
   void SentencePiece::reset_vocabulary()
   {
+#ifdef SP_HAS_VOCAB_RESTRICTION
     _processor.ResetVocabulary();
+#else
+    throw std::runtime_error("The project was built against a SentencePiece version "
+                             "that does not support vocabulary restriction");
+#endif
   }
 
   void SentencePiece::enable_regularization(int nbest_size, float alpha)