diff --git a/supplemental/tokenizer/cpp/en/about.html b/supplemental/tokenizer/cpp/en/about.html index 53af186c..48e76974 100644 --- a/supplemental/tokenizer/cpp/en/about.html +++ b/supplemental/tokenizer/cpp/en/about.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/about_8md.html b/supplemental/tokenizer/cpp/en/about_8md.html index 8a4e1984..17e8f36c 100644 --- a/supplemental/tokenizer/cpp/en/about_8md.html +++ b/supplemental/tokenizer/cpp/en/about_8md.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h.html b/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h.html index 8e3e37f5..03217374 100644 --- a/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h.html +++ b/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
@@ -216,6 +216,9 @@ int AILIA_API ailiaTokenizerGetVocab (struct AILIATokenizer *net, int token, const char **vocab)  Perform encode. More...
  +int AILIA_API ailiaTokenizerAddSpecialTokens (struct AILIATokenizer *net, const char **tokens, unsigned int count) + Add SpecialToken. More...
+  void AILIA_API ailiaTokenizerDestroy (struct AILIATokenizer *net)  It destroys the tokenizer instance. More...
  @@ -537,6 +540,52 @@

Function Documentation

+ +

◆ ailiaTokenizerAddSpecialTokens()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
int AILIA_API ailiaTokenizerAddSpecialTokens (struct AILIATokenizer * net,
const char ** tokens,
unsigned int count 
)
+
+ +

Add SpecialToken.

+
Parameters
+ + + + +
netA tokenizer instance pointer
tokensToken(UTF8)
countThe number of tokens
+
+
+
Returns
If this function is successful, it returns AILIA_STATUS_SUCCESS , or an error code otherwise.
+

This is valid only for AILIA_TOKENIZER_TYPE_ROBERTA and AILIA_TOKENIZER_TYPE_ROBERTA.

+ +
+

◆ ailiaTokenizerCreate()

diff --git a/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h_source.html b/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h_source.html index 3b2a93ca..22283dcd 100644 --- a/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h_source.html +++ b/supplemental/tokenizer/cpp/en/ailia__tokenizer_8h_source.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
@@ -186,36 +186,39 @@
662 int AILIA_API
663 ailiaTokenizerGetVocab(struct AILIATokenizer* net, int token, const char **vocab);
664 
-
674 void AILIA_API ailiaTokenizerDestroy(struct AILIATokenizer* net);
-
675 
-
676 /****************************************************************
-
677  * Utility API
-
678  **/
-
679 
-
699 int AILIA_API ailiaTokenizerUtf8ToUtf32(unsigned int* utf32, unsigned int* processed_byte, const char* utf8, unsigned int utf8_len);
-
700 
-
718 int AILIA_API ailiaTokenizerUtf32ToUtf8(char* utf8, unsigned int* processed_byte, unsigned int utf32);
-
719 
-
720 #ifdef UNICODE
-
721 #define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileW
-
722 #define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileW
-
723 #define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileW
-
724 #define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileW
-
725 #define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileW
-
726 #define ailiaTokenizerOpenConfigFile ailiaTokenizerOpenConfigFileW
-
727 #else
-
728 #define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileA
-
729 #define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileA
-
730 #define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileA
-
731 #define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileA
-
732 #define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileA
-
733 #define ailiaTokenizerOpenTokenizerConfigFile ailiaTokenizerOpenTokenizerConfigFileA
-
734 #endif
-
735 
-
736 #ifdef __cplusplus
-
737 }
-
738 #endif
-
739 #endif /* !defined(INCLUDED_AILIA_TOKENIZER) */
+
686 int AILIA_API
+
687 ailiaTokenizerAddSpecialTokens(struct AILIATokenizer* net, const char **tokens, unsigned int count);
+
688 
+
698 void AILIA_API ailiaTokenizerDestroy(struct AILIATokenizer* net);
+
699 
+
700 /****************************************************************
+
701  * Utility API
+
702  **/
+
703 
+
723 int AILIA_API ailiaTokenizerUtf8ToUtf32(unsigned int* utf32, unsigned int* processed_byte, const char* utf8, unsigned int utf8_len);
+
724 
+
742 int AILIA_API ailiaTokenizerUtf32ToUtf8(char* utf8, unsigned int* processed_byte, unsigned int utf32);
+
743 
+
744 #ifdef UNICODE
+
745 #define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileW
+
746 #define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileW
+
747 #define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileW
+
748 #define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileW
+
749 #define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileW
+
750 #define ailiaTokenizerOpenConfigFile ailiaTokenizerOpenConfigFileW
+
751 #else
+
752 #define ailiaTokenizerOpenModelFile ailiaTokenizerOpenModelFileA
+
753 #define ailiaTokenizerOpenDictionaryFile ailiaTokenizerOpenDictionaryFileA
+
754 #define ailiaTokenizerOpenVocabFile ailiaTokenizerOpenVocabFileA
+
755 #define ailiaTokenizerOpenMergeFile ailiaTokenizerOpenMergeFileA
+
756 #define ailiaTokenizerOpenAddedTokensFile ailiaTokenizerOpenAddedTokensFileA
+
757 #define ailiaTokenizerOpenTokenizerConfigFile ailiaTokenizerOpenTokenizerConfigFileA
+
758 #endif
+
759 
+
760 #ifdef __cplusplus
+
761 }
+
762 #endif
+
763 #endif /* !defined(INCLUDED_AILIA_TOKENIZER) */
ailiaTokenizerGetTextLength
int AILIA_API ailiaTokenizerGetTextLength(struct AILIATokenizer *net, unsigned int *len)
Gets the size of text. (Include null)
ailiaTokenizerOpenAddedTokensFileA
int AILIA_API ailiaTokenizerOpenAddedTokensFileA(struct AILIATokenizer *net, const char *path)
Open added tokens file.
@@ -247,6 +250,7 @@
ailiaTokenizerOpenMergeFileW
int AILIA_API ailiaTokenizerOpenMergeFileW(struct AILIATokenizer *net, const wchar_t *path)
ailiaTokenizerGetCharEnds
int AILIA_API ailiaTokenizerGetCharEnds(struct AILIATokenizer *net, int *char_ends, unsigned int count)
Gets the Char end positions.
ailiaTokenizerDecodeWithSpecialTokens
int AILIA_API ailiaTokenizerDecodeWithSpecialTokens(struct AILIATokenizer *net, const int *tokens, unsigned int token_count)
Perform decode with special tokens.
+
ailiaTokenizerAddSpecialTokens
int AILIA_API ailiaTokenizerAddSpecialTokens(struct AILIATokenizer *net, const char **tokens, unsigned int count)
Add SpecialToken.
Generated by   diff --git a/supplemental/tokenizer/cpp/en/api.html b/supplemental/tokenizer/cpp/en/api.html index 979df2c3..770b0a91 100644 --- a/supplemental/tokenizer/cpp/en/api.html +++ b/supplemental/tokenizer/cpp/en/api.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/api_8md.html b/supplemental/tokenizer/cpp/en/api_8md.html index d2924b9f..05d50983 100644 --- a/supplemental/tokenizer/cpp/en/api_8md.html +++ b/supplemental/tokenizer/cpp/en/api_8md.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/dir_54dc29b291de63e3c0568d35198e740b.html b/supplemental/tokenizer/cpp/en/dir_54dc29b291de63e3c0568d35198e740b.html index a3057ce9..c8c9492b 100644 --- a/supplemental/tokenizer/cpp/en/dir_54dc29b291de63e3c0568d35198e740b.html +++ b/supplemental/tokenizer/cpp/en/dir_54dc29b291de63e3c0568d35198e740b.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/dir_be9381d9fa66b9ba3c9eb52b50213913.html b/supplemental/tokenizer/cpp/en/dir_be9381d9fa66b9ba3c9eb52b50213913.html index 50b9f3a0..84bb9d0a 100644 --- a/supplemental/tokenizer/cpp/en/dir_be9381d9fa66b9ba3c9eb52b50213913.html +++ b/supplemental/tokenizer/cpp/en/dir_be9381d9fa66b9ba3c9eb52b50213913.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/dir_df511e5bd85cec96854b39d5e1c27aa8.html b/supplemental/tokenizer/cpp/en/dir_df511e5bd85cec96854b39d5e1c27aa8.html index 14a51dc0..083aa757 100644 --- a/supplemental/tokenizer/cpp/en/dir_df511e5bd85cec96854b39d5e1c27aa8.html +++ b/supplemental/tokenizer/cpp/en/dir_df511e5bd85cec96854b39d5e1c27aa8.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/dir_e048eec7d1e763bfddf1d1355dfb8d1e.html b/supplemental/tokenizer/cpp/en/dir_e048eec7d1e763bfddf1d1355dfb8d1e.html index 4e3b235c..f93980ee 100644 --- a/supplemental/tokenizer/cpp/en/dir_e048eec7d1e763bfddf1d1355dfb8d1e.html +++ b/supplemental/tokenizer/cpp/en/dir_e048eec7d1e763bfddf1d1355dfb8d1e.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/feature.html b/supplemental/tokenizer/cpp/en/feature.html index e4971497..5bbaf7ad 100644 --- a/supplemental/tokenizer/cpp/en/feature.html +++ b/supplemental/tokenizer/cpp/en/feature.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/feature_8md.html b/supplemental/tokenizer/cpp/en/feature_8md.html index 90b04938..cea939ec 100644 --- a/supplemental/tokenizer/cpp/en/feature_8md.html +++ b/supplemental/tokenizer/cpp/en/feature_8md.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/files.html b/supplemental/tokenizer/cpp/en/files.html index a1b6fa1c..9d1afc99 100644 --- a/supplemental/tokenizer/cpp/en/files.html +++ b/supplemental/tokenizer/cpp/en/files.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
diff --git a/supplemental/tokenizer/cpp/en/globals.html b/supplemental/tokenizer/cpp/en/globals.html index 355069d9..6f49c70b 100644 --- a/supplemental/tokenizer/cpp/en/globals.html +++ b/supplemental/tokenizer/cpp/en/globals.html @@ -22,7 +22,7 @@
ailia_tokenizer -  1.3.1.0 +  1.4.0.0
@@ -107,6 +107,9 @@

- a -