Skip to content

Commit

Permalink
fix for using/building dlls (#47)
Browse files Browse the repository at this point in the history
* fix for using/building dlls

* specify onmt directory when including export header
  • Loading branch information
jhnwnd authored and guillaumekln committed Oct 11, 2018
1 parent 2022477 commit 9af8e2b
Show file tree
Hide file tree
Showing 14 changed files with 57 additions and 29 deletions.
17 changes: 12 additions & 5 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ endif()

set(INCLUDE_DIRECTORIES
${CMAKE_CURRENT_SOURCE_DIR}/include
${PROJECT_BINARY_DIR}
)

set(PUBLIC_HEADERS
Expand Down Expand Up @@ -91,6 +92,10 @@ else()
endif()

add_library(${PROJECT_NAME} ${SOURCES})
include(GNUInstallDirs)
include(GenerateExportHeader)
string(TOLOWER ${PROJECT_NAME} PROJECT_NAME_LOWER)
generate_export_header(${PROJECT_NAME} EXPORT_FILE_NAME ${PROJECT_BINARY_DIR}/onmt/${PROJECT_NAME_LOWER}_export.h)
target_include_directories(${PROJECT_NAME} PUBLIC ${INCLUDE_DIRECTORIES})
target_link_libraries(${PROJECT_NAME} ${LINK_LIBRARIES})

Expand All @@ -101,14 +106,16 @@ endif()

install(
TARGETS ${PROJECT_NAME}
ARCHIVE DESTINATION lib/
LIBRARY DESTINATION lib/
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
)
install(FILES
${PUBLIC_HEADERS}
DESTINATION include/onmt/
${PUBLIC_HEADERS} "${PROJECT_BINARY_DIR}/onmt/${PROJECT_NAME_LOWER}_export.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/onmt"
)
install(FILES
include/onmt/unicode/Unicode.h
DESTINATION include/onmt/unicode
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/onmt/unicode"
)
5 changes: 5 additions & 0 deletions cli/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ include_directories(
${Boost_INCLUDE_DIRS}
)

if (WIN32 AND NOT CYGWIN AND NOT Boost_USE_STATIC_LIBS)
add_definitions(-DBOOST_ALL_NO_LIB) #Boost: Tells the config system not to automatically select which libraries to link against
add_definitions(-DBOOST_ALL_DYN_LINK) #Boost: Forces all libraries that have separate source, to be linked as dll's rather than static libraries on Microsoft Windows
endif()

add_executable(tokenize
tokenize.cc
)
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/AnnotatedToken.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

#include <string>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/CaseModifier.h"

namespace onmt
{

class AnnotatedToken
class OPENNMTTOKENIZER_EXPORT AnnotatedToken
{
public:
AnnotatedToken() = default;
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/BPE.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
#include <unordered_set>
#include <vector>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/SubwordEncoder.h"

namespace onmt
{

class BPE: public SubwordEncoder
class OPENNMTTOKENIZER_EXPORT BPE: public SubwordEncoder
{
public:
BPE(const std::string& model_path);
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/BPELearner.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
#include <string>
#include <unordered_map>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/SubwordLearner.h"

namespace onmt
{
class BPELearner: public SubwordLearner
class OPENNMTTOKENIZER_EXPORT BPELearner: public SubwordLearner
{
public:
BPELearner(bool verbose,
Expand Down
4 changes: 3 additions & 1 deletion include/onmt/CaseModifier.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

#include <string>

#include "onmt/opennmttokenizer_export.h"

namespace onmt
{

class CaseModifier
class OPENNMTTOKENIZER_EXPORT CaseModifier
{
public:
enum class Type
Expand Down
4 changes: 3 additions & 1 deletion include/onmt/ITokenizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
#include <string>
#include <unordered_map>

#include "onmt/opennmttokenizer_export.h"

namespace onmt
{

class ITokenizer
class OPENNMTTOKENIZER_EXPORT ITokenizer
{
public:
static const std::string feature_marker;
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/SPMLearner.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
#include <memory>
#include <unordered_map>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/SubwordLearner.h"

namespace onmt
{

class SPMLearner: public SubwordLearner
class OPENNMTTOKENIZER_EXPORT SPMLearner: public SubwordLearner
{
public:
SPMLearner(bool verbose, const std::string& opts, const std::string& input_filename);
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/SentencePiece.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@

#include <sentencepiece_processor.h>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/SubwordEncoder.h"

namespace onmt
{

class SentencePiece: public SubwordEncoder
class OPENNMTTOKENIZER_EXPORT SentencePiece: public SubwordEncoder
{
public:
SentencePiece(const std::string& model_path);
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/SpaceTokenizer.h
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
#pragma once

#include "onmt/opennmttokenizer_export.h"
#include "onmt/ITokenizer.h"

namespace onmt
{

// This Tokenizer simply splits on spaces. Useful when the text was tokenized
// with an external tool.
class SpaceTokenizer: public ITokenizer
class OPENNMTTOKENIZER_EXPORT SpaceTokenizer: public ITokenizer
{
public:
static ITokenizer& get_instance();
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/SubwordEncoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
#include <string>
#include <vector>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/AnnotatedToken.h"

namespace onmt
{

class SubwordEncoder
class OPENNMTTOKENIZER_EXPORT SubwordEncoder
{
public:
virtual ~SubwordEncoder() = default;
Expand Down
4 changes: 3 additions & 1 deletion include/onmt/SubwordLearner.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@
#include <vector>
#include <iostream>

#include "onmt/opennmttokenizer_export.h"

namespace onmt
{
class Tokenizer;

class SubwordLearner
class OPENNMTTOKENIZER_EXPORT SubwordLearner
{
public:
SubwordLearner(bool verbose);
Expand Down
3 changes: 2 additions & 1 deletion include/onmt/Tokenizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@
#include <unordered_map>
#include <set>

#include "onmt/opennmttokenizer_export.h"
#include "onmt/ITokenizer.h"
#include "onmt/SubwordEncoder.h"

namespace onmt
{

// This Tokenizer implements the behaviour of OpenNMT's tools/tokenize.lua.
class Tokenizer: public ITokenizer
class OPENNMTTOKENIZER_EXPORT Tokenizer: public ITokenizer
{
public:
enum class Mode
Expand Down
28 changes: 15 additions & 13 deletions include/onmt/unicode/Unicode.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,24 @@
#include <vector>
#include <string>

#include "onmt/opennmttokenizer_export.h"

namespace onmt
{
namespace unicode
{

typedef int32_t code_point_t;

std::string cp_to_utf8(code_point_t u);
code_point_t utf8_to_cp(const unsigned char* s, unsigned int &l);
OPENNMTTOKENIZER_EXPORT std::string cp_to_utf8(code_point_t u);
OPENNMTTOKENIZER_EXPORT code_point_t utf8_to_cp(const unsigned char* s, unsigned int &l);

std::vector<std::string> split_utf8(const std::string& str, const std::string& sep);
void explode_utf8(const std::string& str,
OPENNMTTOKENIZER_EXPORT std::vector<std::string> split_utf8(const std::string& str, const std::string& sep);
OPENNMTTOKENIZER_EXPORT void explode_utf8(const std::string& str,
std::vector<std::string>& chars,
std::vector<code_point_t>& code_points);

size_t utf8len(const std::string& str);
OPENNMTTOKENIZER_EXPORT size_t utf8len(const std::string& str);

enum _type_letter
{
Expand All @@ -28,15 +30,15 @@ namespace onmt
_letter_upper
};

bool is_separator(code_point_t u);
bool is_letter(code_point_t u, _type_letter &tl);
bool is_letter(code_point_t u);
bool is_number(code_point_t u);
bool is_mark(code_point_t u);
OPENNMTTOKENIZER_EXPORT bool is_separator(code_point_t u);
OPENNMTTOKENIZER_EXPORT bool is_letter(code_point_t u, _type_letter &tl);
OPENNMTTOKENIZER_EXPORT bool is_letter(code_point_t u);
OPENNMTTOKENIZER_EXPORT bool is_number(code_point_t u);
OPENNMTTOKENIZER_EXPORT bool is_mark(code_point_t u);

_type_letter get_case(code_point_t u);
code_point_t get_upper(code_point_t u);
code_point_t get_lower(code_point_t u);
OPENNMTTOKENIZER_EXPORT _type_letter get_case(code_point_t u);
OPENNMTTOKENIZER_EXPORT code_point_t get_upper(code_point_t u);
OPENNMTTOKENIZER_EXPORT code_point_t get_lower(code_point_t u);

}
}

0 comments on commit 9af8e2b

Please sign in to comment.