docs: Added new wiki tests.

examples/LangChain.Samples.Serve/Program.cs

@@ -20,7 +20,7 @@
 {
     Temperature = 0,
     Stop = ["User:"],
-}), "mistral:latest");
+}), "llama3.1");
 
 
 // 3. Optional. Add custom name generator

src/Helpers/GenerateDocs/Program.cs

@@ -93,6 +93,8 @@ await File.WriteAllTextAsync(newPath, $@"```csharp
         }
     }
 
+    markdown = anyComment ? @"`Scroll till the end of the page if you just want code`  
+" + markdown : markdown;
     markdown += anyComment ? @$"
 # Complete code
 ```csharp

src/Meta/test/WikiTests.AgentWithOllamaReact.cs

@@ -34,7 +34,7 @@ public async Task AgentWithOllamaReact()
         //         Stop = new[] { "Observation", "[END]" }, // add injection word `Observation` and `[END]` to stop the model(just as additional safety feature)
         //         Temperature = 0
         //     });
-        // var llm = new OllamaChatModel(provider, id: "mistral:latest").UseConsoleForDebug();
+        // var llm = new OllamaChatModel(provider, id: "llama3.1").UseConsoleForDebug();
         var apiKey =
             Environment.GetEnvironmentVariable("OPENAI_API_KEY") ??
             throw new InvalidOperationException("OPENAI_API_KEY key is not set");

src/Meta/test/WikiTests.ImageGenerationWithOllamaAndStableDiffusion.cs

@@ -12,13 +12,45 @@ public partial class WikiTests
     [Test]
     public async Task ImageGenerationWithOllamaAndStableDiffusion()
     {
+        //// # Prerequisites
+        //// For this tutorial you would need Ollama and AUTOMATIC1111 locally installed. You can easily find instructions online how to do it for your system.
+        //// I'm using Docker images for both of those.
+        //// Also you would need about 32GB of RAM installed in your PC since two models are "eating" a lot of it.
+        //// 
+        //// # The problem
+        //// Making a prompt for image generation models like Stable Diffusion is not a simple task. Instead of just asking what you need in simple sentence, you would need to describe all the small details about object and environment using quite big set of keywords.
+        //// 
+        //// Let's try to solve this problem!
+        //// 
+        //// # Setup
+        //// 
+        //// Create new console app and add nuget packages:
+        //// ```
+        //// LangChain.Core
+        //// LangChain.Providers.Automatic1111
+        //// LangChain.Providers.Ollama
+        //// ```
+        //// Now we are ready to code!
+        //// 
+        //// # Creating models
+        //// 
+        //// ## Ollama model
+        //// We will use latest version of `llama3.1` for our task. If you don't have mistral yet - it will be downloaded.
+
         var provider = new OllamaProvider(
             options: new RequestOptions
             {
-                Stop = new[] { "\n" },
-                Temperature = 0
+                Temperature = 0,
             });
-        var llm = new OllamaChatModel(provider, id: "mistral:latest").UseConsoleForDebug();
+        var llm = new OllamaChatModel(provider, id: "llama3.1").UseConsoleForDebug();
+
+        //// Here we are stopping generation after `\n` symbol appears. Mistral will put a new line(`\n`) symbol after prompt is generated.
+        //// We are using Temperature of 0 to always have the same result for the same prompt.
+        //// assigning events to model is a good way to see what is going on.
+        //// 
+        //// ## Stable diffusion model
+        //// 
+        //// You can select your model in Automatic1111 UI.
 
         var sdmodel = new Automatic1111Model
         {
@@ -32,6 +64,15 @@ public async Task ImageGenerationWithOllamaAndStableDiffusion()
             },
         };
 
+        //// You should be familiar with these parameters if you were using SD before. But in simple words we're asking SD to generate a portrait without anything bad on it.
+        //// 
+        //// # Prompt
+        //// At this point we are ready to start our LLM magic.  
+        //// We will be using a [special prompting technique](https://www.promptingguide.ai/techniques/fewshot) to explain our expectations to mistral.  
+        //// I took it from [here](https://github.com/vicuna-tools/Stablediffy/blob/main/Stablediffy.txt) with some minor modifications.  
+        //// Basically, we are showing some examples so model could understand a principle of prompt generation. You can play around with examples and instructions to better match your preferences.  
+        //// Now let's build a chain!  
+
         var template =
     @"[INST]Transcript of a dialog, where the User interacts with an Assistant named Stablediffy. Stablediffy knows much about prompt engineering for stable diffusion (an open-source image generation software). The User asks Stablediffy about prompts for stable diffusion Image Generation. 
 
@@ -61,5 +102,7 @@ public async Task ImageGenerationWithOllamaAndStableDiffusion()
 
         // run the chain
         await chain.RunAsync();
+
+        //// If everything done correctly - you should have `image.png` in your bin directory.
     }
 }

src/Meta/test/WikiTests.RagWithOpenAiOllama.cs

@@ -4,6 +4,7 @@
 using LangChain.Providers.OpenAI.Predefined;
 using LangChain.DocumentLoaders;
 using LangChain.Providers.Ollama;
+using LangChain.Providers.OpenAI;
 using LangChain.Splitters.Text;
 using Ollama;
 using static LangChain.Chains.Chain;
@@ -16,28 +17,86 @@ public partial class WikiTests
     [Test]
     public async Task RagWithOpenAiOllama()
     {
+        //// # Introduction
+        //// This tutorial will help you setup a RAG (retrieval-augmented generation) application, where you can ask questions of a PDF document, such as the book [Harry Potter and the Philosopher's Stone](https://canonburyprimaryschool.co.uk/wp-content/uploads/2016/01/Joanne-K.-Rowling-Harry-Potter-Book-1-Harry-Potter-and-the-Philosophers-Stone-EnglishOnlineClub.com_.pdf).
+        //// 
+        //// # RAG
+        //// The principle behind RAG is as follows.
+        //// 
+        //// First, you index the documents of interest.
+        //// - Split your document into smaller text snippets or chunks
+        //// - Generate an embedding for each chunk.  An embedding is a vector (array of numbers) that encodes the semantic meaning of the text
+        //// - Save these embeddings into a special-purpose database, which enables similarity search
+        //// 
+        //// Second, you ask your question.
+        //// - Generate an embedding for the question text
+        //// - Perform a similarity search in your database using that embedding, which will return chunks that are semantically related to the question
+        //// - Pass the chunks and the question to the LLM, along with other prompt text for guidance
+        //// - Receive a nicely-worded response from the LLM that hopefully answers the question
+        //// - PROFIT!
+        //// 
+        //// # Project
+        //// To get started, create a new console application and add the following nuget packages.  (Use the pre-release checkbox.)
+        //// ```
+        //// LangChain
+        //// LangChain.Providers.Ollama
+        //// LangChain.Databases.Sqlite
+        //// LangChain.Sources.Pdf
+        //// ```
+        //// The code here assumes you are using .NET 8.
+        //// 
+        //// # Models
+        //// 
+        //// Lets configure the models.
+        //// 
+        //// ## Completion and Embeddings
+        //// You can use either OpenAI (online) or Ollama (offline):
+        //// 
+        //// ### OpenAI
+        //// To use this chat and embedding model, you will need an API key from OpenAI.  This has non-zero cost.
+
         // prepare OpenAI embedding model
-        var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new InvalidOperationException("OpenAI API key is not set");
-        var embeddingModel = new TextEmbeddingV3SmallModel(apiKey);
+        var provider = new OpenAiProvider(apiKey:
+            Environment.GetEnvironmentVariable("OPENAI_API_KEY") ??
+            throw new InvalidOperationException("OPENAI_API_KEY key is not set"));
+        var embeddingModel = new TextEmbeddingV3SmallModel(provider);
+        var llm = new OpenAiLatestFastChatModel(provider);
+
+        //// ### Ollama
+        //// To use this chat and embedding model, you will need an Ollama instance running.
+        //// This is free, assuming it is running locally--this code assumes it is available at https://localhost:11434.
 
         // prepare Ollama with mistral model
-        var provider = new OllamaProvider(
+        var providerOllama = new OllamaProvider(
             options: new RequestOptions
             {
                 Stop = ["\n"],
                 Temperature = 0.0f,
             });
-        var llm = new OllamaChatModel(provider, id: "mistral:latest").UseConsoleForDebug();
+        var embeddingModelOllama = new OllamaEmbeddingModel(providerOllama, id: "nomic-embed-text");
+        var llmOllama = new OllamaChatModel(providerOllama, id: "llama3.1").UseConsoleForDebug();
+
+        //// Configure the vector database.
 
         using var vectorDatabase = new SqLiteVectorDatabase("vectors.db");
         var vectorCollection = await vectorDatabase.AddDocumentsFromAsync<PdfPigPdfLoader>(
             embeddingModel,
             dimensions: 1536, // Should be 1536 for TextEmbeddingV3SmallModel
+            // First, specify the source to index.
             dataSource: DataSource.FromPath("E:\\AI\\Datasets\\Books\\Harry-Potter-Book-1.pdf"),
             collectionName: "harrypotter",
+            // Second, configure how to extract chunks from the bigger document.
             textSplitter: new RecursiveCharacterTextSplitter(
-                chunkSize: 200,
-                chunkOverlap: 50));
+                chunkSize: 500, // To pick the chunk size, estimate how much information would be required to capture most passages you'd like to ask questions about.  Too many characters makes it difficult to capture semantic meaning, and too few characters means you are more likely to split up important points that are related. In general, 200-500 characters is good for stories without complex sequences of actions.
+                chunkOverlap: 200)); // To pick the chunk overlap you need to estimate the size of the smallest piece of information. It may happen that one chunk ends with `Ron's hair` and the other one starts with `is red`.In this case, an embedding would miss important context, and not be generated propperly. With overlap the end of the first chunk will appear in the begining of the other, eliminating the problem.
+
+        //// # Execution
+        //// 
+        //// Now we will put together the chain of actions that will run whenever we have questions about the document.
+        //// 
+        //// ## Prompt
+        //// 
+        //// Chains require a specially-crafted prompt. For our task we will use the one below. Feel free to tweak this for your purposes.
 
         string promptText =
             @"Use the following pieces of context to answer the question at the end. If the answer is not in context then just say that you don't know, don't try to make up an answer. Keep the answer as short as possible.
@@ -47,16 +106,62 @@ public async Task RagWithOpenAiOllama()
 Question: {question}
 Helpful Answer:";
 
+        //// Inside the prompt we have 2 placeholders: `context` and `question`, which will be replaced during execution.
+        //// 
+        //// - question: The question asked by the user
+        //// - context: The document chunks that were found to be semantically related to the question being asked
+        //// 
+        //// ## Chain
+        //// 
+        //// Now we can configure the chain of actions that should occur during execution.
 
         var chain =
-            Set("Who was drinking a unicorn blood?", outputKey: "question")                // set the question
-            | RetrieveDocuments(vectorCollection, embeddingModel, inputKey: "question", outputKey: "documents", amount: 5) // take 5 most similar documents
-            | StuffDocuments(inputKey: "documents", outputKey: "context")                       // combine documents together and put them into context
-            | Template(promptText)                                                              // replace context and question in the prompt with their values
-            | LLM(llm);                                                                       // send the result to the language model
+            Set("Who was drinking a unicorn blood?", outputKey: "question")     // set the question
+            | RetrieveDocuments(
+                vectorCollection,
+                embeddingModel,
+                inputKey: "question",
+                outputKey: "documents",
+                amount: 5)                                                      // take 5 most similar documents
+            | StuffDocuments(inputKey: "documents", outputKey: "context")       // combine documents together and put them into context
+            | Template(promptText)                                              // replace context and question in the prompt with their values
+            | LLM(llm);                                                         // send the result to the language model
 
-        var result = await chain.RunAsync("text", CancellationToken.None);                                        // get chain result
+        // get chain result
+        var result = await chain.RunAsync("text", CancellationToken.None);
 
         Console.WriteLine(result);
+
+        //// We are done! Since we previously registered for events on the completion model, the output will be printed automatically.
+        //// 
+        //// # Example
+        //// 
+        //// Here is an example execution output.
+        //// 
+        //// - The first paragraph is part of our prompt template.
+        //// - The next 5 paragraphs are semantically-related context.
+        //// - The "Question:" line shows the question we asked.
+        //// - The "Helpful Answer:" line shows how the completion model answered the question we asked.
+        //// 
+        //// ```
+        //// Use the following pieces of context to answer the question at the end. If the answer is not in context then just say that you don't know, don't try to make up an answer. Keep the answer as short as possible.
+        //// 
+        //// and began to drink its blood. 'AAAAAAAAAAARGH!' Malfoy let out a terrible scream and bolted - so did Fang. The hooded figure raised its head and looked right at Harry - unicorn blood was dribbling
+        //// 
+        //// close by. There were still spots of unicorn blood here and there along the winding path.
+        //// 
+        //// is because it is a monstrous thing, to slay a unicorn,' said Firenze. 'Only one who has nothing to lose, and everything to gain, would commit such a crime. The blood of a unicorn will keep you alive,
+        //// 
+        //// beast. Harry, Malfoy and Fang stood transfixed. The cloaked figure reached the uni-corn, it lowered its head over the wound in the animal's side, and began to drink its blood. 'AAAAAAAAAAARGH!'
+        //// 
+        //// light breeze lifted their hair as they looked into the Forest. 'Look there,' said Hagrid, 'see that stuff shinin' on the ground? Silvery stuff? That's unicorn blood. There's a unicorn in there bin
+        //// 
+        //// Question: Who was drinking a unicorn blood?
+        //// Helpful Answer: The hooded figure was drinking a unicorn's blood.
+        //// ```
+        //// 
+        //// As you can see, every piece of context is mentioning unicorn blood. In the first piece the hooded figure is mentioned. So the model takes this as an answer to our question.
+        //// 
+        //// Now go and try to ask your own questions!
     }
 }

src/Meta/test/WikiTests.UsingChainOutput.cs

@@ -10,6 +10,9 @@ public partial class WikiTests
     [Test]
     public async Task UsingChainOutput()
     {
+        //// # Setup
+        //// We will take the code from [Getting started] tutorial as our starting point.
+
         // get model path
         var modelPath = await HuggingFaceModelDownloader.GetModelAsync(
             repository: "TheBloke/Thespis-13B-v0.5-GGUF",
@@ -25,13 +28,32 @@ You are an AI assistant that greets the world.
 World: Hello, Assistant!
 Assistant:";
 
+        //// # Getting the chain output
+        //// 
+        //// Almost every possible link in a chain are having having at least one input and output.
+        //// 
+        //// Look here:
+
         var chain =
             Set(prompt, outputKey: "prompt")
             | LLM(model, inputKey: "prompt", outputKey: "result");
+
+        //// This means that, after link `Set` get executed, we are storring it's result into "prompt" variable inside of chain context.
+        //// In its turn, link `LLM` gets "prompt" variable from chain context and uses it's as input.
+        //// 
+        //// `LLM` link also has output key argument. Let's use it to save the result of llm.
 
         var result = await chain.RunAsync("result", CancellationToken.None);
-        Console.WriteLine("---");
+
+        //// Now the `LLM` link saves it's result into "result" variable inside of chain context. But how do we extract it from there?
+        //// 
+        //// `chain.Run()` method has an optional argument "resultKey". This allows you to specify variable inside of chain context to return as a result.
+
         Console.WriteLine(result);
-        Console.WriteLine("---");
+
+        //// Output:
+        //// ```
+        //// Hello, World! How can I help you today?
+        //// ```
     }
 }