Merge branch 'master' into user/t-tonmoneta/CS_Sample_app_refactord

S3/S3EventGridPoc/Function.cs

@@ -0,0 +1,93 @@
+using Amazon.Lambda.Core;
+using Amazon.Lambda.S3Events;
+using Amazon.S3;
+using Amazon.S3.Util;
+using Kusto.Data;
+using Kusto.Data.Net.Client;
+using Kusto.Ingest;
+
+// Assembly attribute to enable the Lambda function's JSON input to be converted into a .NET class.
+[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))]
+
+namespace S3EventGridPoc;
+
+public class Function
+{
+    private readonly IKustoIngestClient m_client;
+
+    IAmazonS3 S3Client { get; set; }
+
+    /// <summary>
+    /// Default constructor. This constructor is used by Lambda to construct the instance.
+    /// </summary>
+    public Function()
+    {
+        S3Client = new AmazonS3Client();
+        string appId = Environment.GetEnvironmentVariable("AppId");
+        string appKey = Environment.GetEnvironmentVariable("AppKey");
+        string authority = Environment.GetEnvironmentVariable("AppTenant");
+        string clusterUri = Environment.GetEnvironmentVariable("IngestionUri");
+        var kustoConnectionStringBuilderDM =
+            new KustoConnectionStringBuilder(clusterUri)
+            .WithAadApplicationKeyAuthentication(appId, appKey, authority);
+
+        Console.WriteLine($"Initializing an ingest client for {clusterUri}");
+        // Create an ingest client
+        // Note, that creating a separate instance per ingestion operation is an anti-pattern.
+        // IngestClient classes are thread-safe and intended for reuse
+        m_client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
+    }
+
+    /// <summary>
+    /// Constructs an instance with a preconfigured S3 client. This can be used for testing the outside of the Lambda environment.
+    /// </summary>
+    /// <param name="s3Client"></param>
+    public Function(IAmazonS3 s3Client)
+    {
+        this.S3Client = s3Client;
+
+        string appId = Environment.GetEnvironmentVariable("AppId");
+        string appKey = Environment.GetEnvironmentVariable("AppKey");
+        string authority = Environment.GetEnvironmentVariable("AppTenant");
+        string clusterUri = Environment.GetEnvironmentVariable("IngestionUri");
+        var kustoConnectionStringBuilderDM =
+            new KustoConnectionStringBuilder(clusterUri)
+            .WithAadApplicationKeyAuthentication(appId, appKey, authority);
+
+        Console.WriteLine($"Initializing an ingest client for {clusterUri}");
+        // Create an ingest client
+        // Note, that creating a separate instance per ingestion operation is an anti-pattern.
+        // IngestClient classes are thread-safe and intended for reuse
+        m_client = KustoIngestFactory.CreateQueuedIngestClient(kustoConnectionStringBuilderDM);
+
+    }
+
+    /// <summary>
+    /// This method is called for every Lambda invocation. This method takes in an S3 event object and can be used 
+    /// to respond to S3 notifications.
+    /// </summary>
+    public async Task FunctionHandler(S3Event evnt, ILambdaContext context)
+    {
+        var awsCredentials = Environment.GetEnvironmentVariable("AwsCredentials");
+        string table = Environment.GetEnvironmentVariable("TargetTable");
+        string database = Environment.GetEnvironmentVariable("TargetDatabase");
+
+        foreach (var record in evnt.Records)
+        {
+            var s3 = record.S3;
+            Console.WriteLine($"[{record.AwsRegion} - {record.EventTime}] Bucket = {s3.Bucket.Name}, Key = {s3.Object.Key}");
+
+            // Ingest from blobs according to the required properties
+            var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: database, tableName: table)
+            {
+                FlushImmediately = true
+            };
+
+            var sourceOptions = new StorageSourceOptions() { DeleteSourceOnSuccess = false, Size = s3.Object.Size };
+            var uri = $"https://{s3.Bucket.Name}.s3.{record.AwsRegion}.amazonaws.com/{s3.Object.Key}";
+            Console.WriteLine($"start to ingest {uri}");
+            await m_client.IngestFromStorageAsync(uri:$"{uri};AwsCredentials={awsCredentials}", ingestionProperties: kustoIngestionProperties, sourceOptions);
+            Console.WriteLine($"complete to ingest {uri}");
+        }
+    }
+}

S3/S3EventGridPoc/Readme.md

@@ -0,0 +1,23 @@
+# Automatic Ingestion from S3 to ADX using lambda
+## Deploy the lambda
+Find instructions [here]( https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-package.html)
+## Define AAD application
+* Find instructions [here]( https://docs.microsoft.com/azure/active-directory/develop/howto-create-service-principal-portal)
+* Define a key for the application and save it.
+* Grant the application permissions on your ADX cluster - at least database Ingestor on the database you want to ingest to.
+## Define environment variables
+* Under Lambda -> Configuration -> Environment variables define the following
+*   AppKey - Your AAD app key
+*	AppId - Your AAD app Id
+*	AppTenant - Your aad app tenant
+*   IngestionUri - URL to ingest to your ADX cluster. (https://ingest-{clusterName}.{region}.kusto.windows.net)
+*	TargetDatabase - ADX database to ingest to.
+*	TargetTable - Table to ingest to.
+*	AwsCredentials - Key and secret of AWS account which has permissions on the bucket, separated by comma.
+## Define S3 trigger
+*	Go to your S3 bucket -> Properties
+*	Scroll down to Event notifications and click on Create event notification
+*	Define the events you want to notify (e.g. all creation events)
+*	Choose your lambda as the destination
+## Done
+Now whenever you put a blob into this bucket, it will be automatically ingested into your ADX!

S3/S3EventGridPoc/S3EventGridPoc.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
+    <AWSProjectType>Lambda</AWSProjectType>
+    <!-- This property makes the build directory similar to a publish directory and helps the AWS .NET Lambda Mock Test Tool find project dependencies. -->
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+    <!-- Generate ready to run images during publishing to improve cold start time. -->
+    <PublishReadyToRun>true</PublishReadyToRun>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.1.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.3.0" />
+    <PackageReference Include="Amazon.Lambda.S3Events" Version="2.0.1" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.9.3" />
+    <PackageReference Include="Microsoft.Azure.Kusto.Ingest" Version="10.1.0" />
+  </ItemGroup>
+</Project>

S3/S3EventGridPoc/S3EventGridPoc.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.2.32616.157
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "S3EventGridPoc", "S3EventGridPoc.csproj", "{6D84D67C-8157-4694-9B37-71C70B217DFF}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{6D84D67C-8157-4694-9B37-71C70B217DFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6D84D67C-8157-4694-9B37-71C70B217DFF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6D84D67C-8157-4694-9B37-71C70B217DFF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6D84D67C-8157-4694-9B37-71C70B217DFF}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {8666E49C-956C-4A5A-B599-F010E80C8D3E}
+	EndGlobalSection
+EndGlobal

S3/S3EventGridPoc/aws-lambda-tools-defaults.json

@@ -0,0 +1,15 @@
+{
+  "Information": [
+    "This file provides default values for the deployment wizard inside Visual Studio and the AWS Lambda commands added to the .NET Core CLI.",
+    "To learn more about the Lambda commands with the .NET Core CLI execute the following command at the command line in the project root directory.",
+    "dotnet lambda help",
+    "All the command line options for the Lambda command can be specified in this file."
+  ],
+  "profile": "",
+  "region": "",
+  "configuration": "Release",
+  "function-runtime": "dotnet6",
+  "function-memory-size": 256,
+  "function-timeout": 30,
+  "function-handler": "S3EventGridPoc::S3EventGridPoc.Function::FunctionHandler"
+}

SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.8 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

client/QuickStart/README.md

@@ -32,13 +32,20 @@ You can use it as a baseline to write your own first kusto client application, a
 1. Open a command line window and navigate to the folder where you extracted the app. 
 2. Either use an IDE of choice to build and run the project, or do the following using the command line window:
    1. For .net Core:
-      1. Run `dotnet build QuickStart` to build the project and its dependencies into a set of binaries, including an executable that can be used to run the application. For more information, click [here](https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build)
+      1. Run `dotnet build QuickStart -o OUTPUT_PATH` to build the project and its dependencies into a set of binaries, including an executable that can be used to run the application. For more information, click [here](https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-build)
    2. For .net Framework:
-      1. Locate your MSBuild app and Run `MSBuild build QuickStart.csproj` to build the project and its dependencies into a set of binaries, including an executable that can be used to run the application. For more information, click [here](https://docs.microsoft.com/en-us/visualstudio/msbuild/msbuild-command-line-reference?view=vs-2022)
+      1. Locate your MSBuild app and Run `MSBuild build QuickStart.csproj /p:OutputPath=OUTPUT_PATH` to build the project and its dependencies into a set of binaries, including an executable that can be used to run the application. For more information, click [here](https://docs.microsoft.com/en-us/visualstudio/msbuild/msbuild-command-line-reference?view=vs-2022)
    3. Make sure the build succeeded without any errors.
    4. Go to your selected output path, and run the exe file from the command line window with `QuickStart.exe` or just double click it from the file explorer.
 
 
 #### Troubleshooting
 
-* If you are having trouble running the app from your IDE, first check if the app runs from the command line, then consult the troubleshooting references of your IDE.
+* If you are having trouble running the app from your IDE, first check if the app runs from the command line, then consult the troubleshooting references of your IDE.
+
+### Optional changes
+
+You can make the following changes from within the app itself:
+
+- Change the default User-Prompt authentication method by editing `AUTHENTICATION_MODE`.
+- Change the app to run without stopping between steps by setting `WAIT_FOR_USER = false`

client/QuickStart/SampleApp.cs

@@ -20,7 +20,9 @@ public enum SourceType
         /// Ingest from local file
         localfilesource,
         /// Ingest from blob
-        blobsource
+        blobsource,
+        /// Do not ingest (ingest from nowhere)
+        nosource
     }
 
     /// <summary>
@@ -444,4 +446,4 @@ private static void WaitForUserToProceed(string promptMsg)
             }
         }
     }
-}
+}