From 5eb614a34e9ff01cbacae2f966c434996d0be926 Mon Sep 17 00:00:00 2001 From: Guillaume Klein Date: Mon, 1 Oct 2018 11:45:12 +0200 Subject: [PATCH] Do not break compilation for users with old SentencePiece versions (#40) --- CHANGELOG.md | 2 ++ CMakeLists.txt | 5 +++++ src/SentencePiece.cc | 10 ++++++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03e9ff19..652cdb30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixes and improvements +* Do not break compilation for users with old SentencePiece versions + ## [v1.9.0](https://github.com/OpenNMT/Tokenizer/releases/tag/v1.9.0) (2018-09-25) ### New features diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c54711a..d7160c82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,11 @@ else() if(HAS_SAMPLE_ENCODE) add_definitions(-DSP_HAS_SAMPLE_ENCODE) endif() + + file(STRINGS ${SP_INCLUDE_DIR}/sentencepiece_processor.h HAS_VOCAB_RESTRICTION REGEX "SetVocabulary") + if(HAS_VOCAB_RESTRICTION) + add_definitions(-DSP_HAS_VOCAB_RESTRICTION) + endif() endif() add_library(${PROJECT_NAME} ${SOURCES}) diff --git a/src/SentencePiece.cc b/src/SentencePiece.cc index 9695cd3f..78cf8361 100644 --- a/src/SentencePiece.cc +++ b/src/SentencePiece.cc @@ -21,12 +21,22 @@ namespace onmt void SentencePiece::set_vocabulary(const std::vector& vocabulary) { +#ifdef SP_HAS_VOCAB_RESTRICTION _processor.SetVocabulary(vocabulary); +#else + throw std::runtime_error("The project was built against a SentencePiece version " + "that does not support vocabulary restriction"); +#endif } void SentencePiece::reset_vocabulary() { +#ifdef SP_HAS_VOCAB_RESTRICTION _processor.ResetVocabulary(); +#else + throw std::runtime_error("The project was built against a SentencePiece version " + "that does not support vocabulary restriction"); +#endif } void SentencePiece::enable_regularization(int nbest_size, float alpha)