Add docstring examples for tokenize() & detokenize()

src/lib.rs

@@ -220,6 +220,31 @@ impl Client {
             .await
     }
 
+    /// Tokenize a prompt for a specific model.
+    ///
+    /// ```no_run
+    /// use aleph_alpha_client::{Client, Error, How, TaskTokenization};
+    ///
+    /// async fn tokenize() -> Result<(), Error> {
+    ///     let client = Client::new(AA_API_TOKEN)?;
+    ///
+    ///     // Name of the model for which we want to tokenize text.
+    ///     let model = "luminous-base";
+    ///
+    ///     // Text prompt to be tokenized.
+    ///     let prompt = "An apple a day";
+    ///
+    ///     let task = TaskTokenization {
+    ///         prompt,
+    ///         tokens: true,       // return text-tokens
+    ///         token_ids: true,    // return numeric token-ids
+    ///     };
+    ///     let respones = client.tokenize(&task, model, &How::default()).await?;
+    ///
+    ///     dbg!(&respones);
+    ///     Ok(())
+    /// }
+    /// ```
     pub async fn tokenize(
         &self,
         task: &TaskTokenization<'_>,
@@ -231,6 +256,29 @@ impl Client {
             .await
     }
 
+    /// Detokenize a list of token ids into a string.
+    ///
+    /// ```no_run
+    /// use aleph_alpha_client::{Client, Error, How, TaskDetokenization};
+    ///
+    /// async fn detokenize() -> Result<(), Error> {
+    ///     let client = Client::new(AA_API_TOKEN)?;
+    ///
+    ///     // Specify the name of the model whose tokenizer was used to generate the input token ids.
+    ///     let model = "luminous-base";
+    ///
+    ///     // Token ids to convert into text.
+    ///     let token_ids: Vec<u32> = vec![556, 48741, 247, 2983];
+    ///
+    ///     let task = TaskDetokenization {
+    ///         token_ids: &token_ids,
+    ///     };
+    ///     let respones = client.detokenize(&task, model, &How::default()).await?;
+    ///
+    ///     dbg!(&respones);
+    ///     Ok(())
+    /// }
+    /// ```
     pub async fn detokenize(
         &self,
         task: &TaskDetokenization<'_>,