Merged PR 3858: FEAT: The SetHeadersElement can be used to convert va…

…lues from all 'SetHeader*' properties into a dictionary that can be used to populate the HTTP response headers

- FEAT: The SetHeadersElement can be used to convert values from all 'SetHeader*' properties into a dictionary that can be used to populate the HTTP response headers. This element is added automatically for web integration pipelines.
- FEAT: When using web integration, the pipeline will now automatically set HTTP headers in the response based on the values of properties starting with 'SetHeader'
  - For ASP.NET  core, setting the response headers is handled by the SetHeadersService
  - For ASP.NET  , setting the response headers is handled by the SetHeadersProvider
- REFACTOR: Combine the separate 'options' configuration classes for ASP.NET  Core and ASP.NET  into a single class. Add an option to flag the 'SetHeader' functionality on/off.
- BUG: Should only try and read from the 'Form' property on an HTTP request after checking if the content type is correct. This addresses #5 .
- BUG: ASP.NET  Framework integration was not reading form parameters. This has now been corrected.
- TEST: Add some tests to verify that the functionality to add form parameters to evidence in the WebRequestEvidenceService is functioning correctly.
- TEST: Add a new JsonBuilderElement test to verify the javascriptProperties element in the JSON for properties with type JavaScript, AspectPropertyValue<JavaScript> or IAspectPropertyValue<JavaScript>.
- TEST: Modify the JsonBuilderElement tests to verify that the delayed evidence functionality works correctly is the property type is JavaScript, AspectPropertyValue<JavaScript> or IAspectPropertyValue<JavaScript>.
- BUG: Correct a bug that stopped properties from being included in the javascriptProperties list if they had type AspectPropertyValue<JavaScript>.
- BUG: JsonBuilderElement was not correctly checking JavaScript types when getting delayed execution properties. This has now been corrected. Also moved the type check to a static utility class.
- DOC: Commented the UseFiftyOne extension method to make it clear that it must be called before any ExceptionHandlerExtensions methods.
- BUILD: Use common-ci template.

Related work items: #4382, #4383

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElement/Templates/javascript"]
 	path = FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElement/Templates/javascript
-	url=../javascript
+	url=../javascript
+[submodule "ci/common-ci"]
+	path = ci/common-ci
+url=../common-ci

FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterAggregator.cs

@@ -54,22 +54,22 @@ public EvidenceKeyFilterAggregator() :
         /// </param>
         public void AddFilter(IEvidenceKeyFilter filter)
         {
-            var whiteListFilter = filter as EvidenceKeyFilterWhitelist;
+            var inclusionListFilter = filter as EvidenceKeyFilterWhitelist;
             bool addFilter = true;
 
-            if (whiteListFilter != null)
+            if (inclusionListFilter != null)
             {
-                // If the filter is a white list filter using the OrdinalIgnoreCase 
-                // comparer then add it's list to this instance's white list to 
-                // give better performance.
-                if (whiteListFilter.Comparer == StringComparer.OrdinalIgnoreCase)
+                // If the filter is an inclusion list filter using the 
+                // OrdinalIgnoreCase comparer then add it's list to this 
+                // instance's inclusion list to give better performance.
+                if (inclusionListFilter.Comparer == StringComparer.OrdinalIgnoreCase)
                 { 
                     addFilter = false;
-                    foreach (var entry in whiteListFilter.Whitelist)
+                    foreach (var entry in inclusionListFilter.Whitelist)
                     {
-                        if (_whitelist.ContainsKey(entry.Key) == false)
+                        if (_inclusionList.ContainsKey(entry.Key) == false)
                         {
-                            _whitelist.Add(entry.Key, entry.Value);
+                            _inclusionList.Add(entry.Key, entry.Value);
                         }
                     }
 
@@ -100,7 +100,7 @@ public void AddFilter(IEvidenceKeyFilter filter)
         /// </returns>
         public override bool Include(string key)
         {
-            // First check the white list as this will be faster than
+            // First check the inclusionList as this will be faster than
             // almost anything else (check against a hash table)
             bool include = base.Include(key);
 

FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterWhitelist.cs

@@ -29,7 +29,7 @@
 namespace FiftyOne.Pipeline.Core.Data
 {
     /// <summary>
-    /// This evidence filter will only include keys that are on a whitelist
+    /// This evidence filter will only include keys that are in a list
     /// that is specified at construction time.
     /// </summary>
     public class EvidenceKeyFilterWhitelist : IEvidenceKeyFilter
@@ -38,33 +38,33 @@ public class EvidenceKeyFilterWhitelist : IEvidenceKeyFilter
         // Extending classes can make direct use of these fields.
 
         /// <summary>
-        /// The dictionary containing all keys in the whitelist and the
-        /// order of precedence.
+        /// The dictionary containing all keys to be included by the filter
+        /// and the order of precedence.
         /// </summary>
-        protected Dictionary<string, int> _whitelist;
+        protected Dictionary<string, int> _inclusionList;
         /// <summary>
         /// The equality comparer that is used to determine if a supplied
-        /// string key is in the whitelist or not.
+        /// string key is in the inclusion list or not.
         /// By default, a case insensitive comparison is used.
         /// </summary>
         protected IEqualityComparer<string> _comparer = 
             StringComparer.OrdinalIgnoreCase;
 #pragma warning restore CA1051 // Do not declare visible instance fields
 
         /// <summary>
-        /// Get the keys in the white list as a read only dictionary.
+        /// Get the keys in the inclusion list as a read only dictionary.
         /// </summary>
         public IReadOnlyDictionary<string, int> Whitelist
         {
             get
             {
-                return new ReadOnlyDictionary<string, int>(_whitelist);
+                return new ReadOnlyDictionary<string, int>(_inclusionList);
             }
         }
 
         /// <summary>
         /// Get the equality comparer that is used to determine if a supplied
-        /// string key is in the whitelist or not.
+        /// string key is in the inclusion list or not.
         /// </summary>
         public IEqualityComparer<string> Comparer => _comparer;
 
@@ -73,52 +73,52 @@ public IReadOnlyDictionary<string, int> Whitelist
         /// The filter will be case-insensitive. For a case-sensitive filter
         /// use the overload that takes an <see cref="IEqualityComparer{T}"/>.
         /// </summary>
-        /// <param name="whitelist">
+        /// <param name="inclusionList">
         /// The list of evidence keys that is filter will include.
         /// By default, all keys will have the same order of precedence.
         /// </param>
-        public EvidenceKeyFilterWhitelist(List<string> whitelist)
+        public EvidenceKeyFilterWhitelist(List<string> inclusionList)
         {
-            PopulateFromList(whitelist);
+            PopulateFromList(inclusionList);
         }
 
         /// <summary>
         /// Constructor
         /// </summary>
-        /// <param name="whitelist">
+        /// <param name="inclusionList">
         /// The list of evidence keys that is filter will include.
         /// By default, all keys will have the same order of precedence.
         /// </param>
         /// <param name="comparer">
         /// Comparator to use when comparing the keys.
         /// </param>
         public EvidenceKeyFilterWhitelist(
-            List<string> whitelist,
+            List<string> inclusionList,
             IEqualityComparer<string> comparer)
         {
             _comparer = comparer;
-            PopulateFromList(whitelist);
+            PopulateFromList(inclusionList);
         }
 
         /// <summary>
         /// Constructor
         /// The filter will be case-insensitive. For a case-sensitive filter
         /// use the overload that takes an <see cref="IEqualityComparer{T}"/>.
         /// </summary>
-        /// <param name="whitelist">
+        /// <param name="inclusionList">
         /// The dictionary of evidence keys that is filter will include.
         /// The order of precedence of each key is given by the value of
         /// the key/value pair.
         /// </param>
-        public EvidenceKeyFilterWhitelist(Dictionary<string, int> whitelist)
+        public EvidenceKeyFilterWhitelist(Dictionary<string, int> inclusionList)
         {
-            PopulateFromDictionary(whitelist);
+            PopulateFromDictionary(inclusionList);
         }
 
         /// <summary>
         /// Constructor
         /// </summary>
-        /// <param name="whitelist">
+        /// <param name="inclusionList">
         /// The dictionary of evidence keys that is filter will include.
         /// The order of precedence of each key is given by the value of
         /// the key/value pair.
@@ -127,22 +127,22 @@ public EvidenceKeyFilterWhitelist(Dictionary<string, int> whitelist)
         /// Comparator to use when comparing the keys.
         /// </param>
         public EvidenceKeyFilterWhitelist(
-            Dictionary<string, int> whitelist,
+            Dictionary<string, int> inclusionList,
             IEqualityComparer<string> comparer)
         {
             _comparer = comparer;
-            PopulateFromDictionary(whitelist);
+            PopulateFromDictionary(inclusionList);
         }
 
 
-        private void PopulateFromList(List<string> whitelist)
+        private void PopulateFromList(List<string> inclusionList)
         {
-            _whitelist = whitelist.ToDictionary(w => w, w => 0, _comparer);
+            _inclusionList = inclusionList.ToDictionary(w => w, w => 0, _comparer);
         }
 
-        private void PopulateFromDictionary(Dictionary<string, int> whitelist)
+        private void PopulateFromDictionary(Dictionary<string, int> inclusionList)
         {
-            _whitelist = whitelist.ToDictionary(w => w.Key, w => w.Value, _comparer);
+            _inclusionList = inclusionList.ToDictionary(w => w.Key, w => w.Value, _comparer);
         }
 
         /// <summary>
@@ -156,7 +156,7 @@ private void PopulateFromDictionary(Dictionary<string, int> whitelist)
         /// </returns>
         public virtual bool Include(string key)
         {
-            return _whitelist.ContainsKey(key);
+            return _inclusionList.ContainsKey(key);
         }
 
         /// <summary>
@@ -168,13 +168,13 @@ public virtual bool Include(string key)
         /// <returns>
         /// The order, where lower values indicate a higher order of 
         /// precedence. 
-        /// Null if the key is not in the white list.
+        /// Null if the key is not in the inclusion list.
         /// </returns>
         public virtual int? Order(string key)
         {
             int? result = 0;
             int temp;
-            if(_whitelist.TryGetValue(key, out temp) == false)
+            if(_inclusionList.TryGetValue(key, out temp) == false)
             {
                 result = null;
             }

FiftyOne.Pipeline.Core/Data/IEvidenceKeyFilter.cs

@@ -54,7 +54,7 @@ public interface IEvidenceKeyFilter
         /// <returns>
         /// The order, where lower values indicate a higher order of 
         /// precedence. 
-        /// Null if the key is not in the white list.
+        /// Null if the key is not recognized.
         /// </returns>
         int? Order(string key);
     }

FiftyOne.Pipeline.Core/FlowElements/PipelineBuilder.cs

@@ -23,6 +23,7 @@
 using FiftyOne.Pipeline.Core.Attributes;
 using FiftyOne.Pipeline.Core.Configuration;
 using FiftyOne.Pipeline.Core.Exceptions;
+using FiftyOne.Pipeline.Core.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using System;
@@ -246,7 +247,8 @@ private void AddElementToList(
             }
 
             object builderInstance = null;
-            if (_services != null)
+            if (_services != null &&
+                _services.GetType().Equals(typeof(FiftyOneServiceProvider)) == false)
             {
                 // Try to get a a builder instance from the service collection.
                 builderInstance = _services.GetRequiredService(builderType);
@@ -434,6 +436,75 @@ private void AddParallelElementsToList(
             elements.Add(parallelInstance);
         }
 
+        /// <summary>
+        /// Get the services required for the constructor, and call it with them.
+        /// </summary>
+        /// <param name="constructor">
+        /// The constructor to call.
+        /// </param>
+        /// <returns>
+        /// Instance returned by the constructor.
+        /// </returns>
+        private object CallConstructorWithServicesForAssemblies(
+            ConstructorInfo constructor)
+        {
+            ParameterInfo[] parameters = constructor.GetParameters();
+            object[] services = new object[parameters.Length];
+            for (int i = 0; i < parameters.Length; i++)
+            {
+                if (parameters[i].ParameterType.Equals(typeof(ILoggerFactory)))
+                {
+                    services[i] = LoggerFactory;
+                }
+                else
+                {
+                    services[i] = _services.GetService(parameters[i].ParameterType);
+                }
+            }
+            return Activator.CreateInstance(constructor.DeclaringType, services);
+        }
+
+
+        /// <summary>
+        /// Get the best constructor for the list of constructors. Best meaning
+        /// the constructor with the most parameters which can be fulfilled.
+        /// </summary>
+        /// <param name="constructors">
+        /// Constructors to get the best of.
+        /// </param>
+        /// <returns>
+        /// Best constructor or null if none have parameters that can be
+        /// fulfilled.
+        /// </returns>
+        private ConstructorInfo GetBestConstructorForAssemblies(
+            IEnumerable<ConstructorInfo> constructors)
+        {
+            ConstructorInfo bestConstructor = null;
+            foreach (var constructor in constructors)
+            {
+                if (bestConstructor == null ||
+                    constructor.GetParameters().Length >
+                    bestConstructor.GetParameters().Length)
+                {
+                    var hasServices = true;
+                    foreach (var param in constructor.GetParameters())
+                    {
+                        if (param.ParameterType.Equals(typeof(ILoggerFactory)) == false &&
+                            _services.GetService(param.ParameterType) == null)
+                        {
+                            hasServices = false;
+                            break;
+                        }
+                    }
+                    if (hasServices == true)
+                    {
+                        bestConstructor = constructor;
+                    }
+                }
+            }
+            return bestConstructor;
+        }
+
         /// <summary>
         /// Instantiate a new builder instance from the assemblies which are
         /// currently loaded.
@@ -450,17 +521,29 @@ private object GetBuilderFromAssemlies(Type builderType)
             var loggerConstructors = builderType.GetConstructors()
                 .Where(c => c.GetParameters().Length == 1 &&
                 c.GetParameters()[0].ParameterType == typeof(ILoggerFactory));
-
+            var serviceConstructors = builderType.GetConstructors()
+                .Where(c => c.GetParameters().Length > 1 &&
+                c.GetParameters().All(p => p.ParameterType.Equals(typeof(ILoggerFactory)) ||
+                (_services != null &&
+                _services.GetService(p.ParameterType) != null)));
+
             if (defaultConstructors.Any() == false &&
-                loggerConstructors.Any() == false)
+                loggerConstructors.Any() == false &&
+                serviceConstructors.Any() == false)
             {
                 return null;
             }
 
             // Create the builder instance using the constructor with a logger
             // factory, or the default constructor if one taking a logger
             // factory is not available.
-            if (loggerConstructors.Any())
+            if (serviceConstructors.Any() &&
+                GetBestConstructorForAssemblies(serviceConstructors) != null)
+            {
+                return CallConstructorWithServicesForAssemblies(
+                    GetBestConstructorForAssemblies(serviceConstructors));
+            }
+            else if (loggerConstructors.Any())
             {
                 return Activator.CreateInstance(builderType, LoggerFactory);
             }