From 1f9bc267c9de8abac31d9c4f6b7140b9ccddc396 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Fri, 23 Apr 2021 15:03:43 +0100 Subject: [PATCH 01/14] FEAT: The SetHeadersElement can be used to convert values from all 'SetHeader*' properties into a dictionary that can be used to populate the HTTP response headers --- .../FlowElement/JsonBuilderElement.cs | 3 +- .../Data/ISetHeadersData.cs | 44 +++ .../Data/SetHeadersData.cs | 69 +++++ .../FlowElements/ISetHeadersElement.cs | 41 +++ .../FlowElements/SetHeadersElement.cs | 260 +++++++++++++++++ .../FlowElements/SetHeadersElementBuilder.cs | 88 ++++++ .../Messages.Designer.cs | 18 ++ .../Messages.resx | 6 + .../PipelineWebIntegrationOptions.cs | 31 +- .../FlowElements/SetHeadersElementTests.cs | 275 ++++++++++++++++++ .../FiftyOne.Pipeline.Web.Framework.csproj | 2 +- .../Providers/SetHeadersProvider.cs | 136 +++++++++ .../WebPipeline.cs | 14 + .../FiftyOne.Pipeline.Web/Constants.cs | 26 ++ .../FiftyOneJSViewComponent.cs | 1 + .../FiftyOneMiddleware.cs | 29 +- .../FiftyOne.Pipeline.Web/FiftyOneStartup.cs | 158 ++++++---- .../PipelineWebIntegrationOptions.cs | 59 ---- .../Services/FiftyOneJSService.cs | 1 + .../Services/ISetHeadersService.cs | 49 ++++ .../Services/SetHeadersService.cs | 132 +++++++++ .../FiftyOne.Pipeline.Web.Tests.csproj | 2 + .../FiftyOneMiddlewareTest.cs | 24 +- 23 files changed, 1343 insertions(+), 125 deletions(-) create mode 100644 FiftyOne.Pipeline.Engines.FiftyOne/Data/ISetHeadersData.cs create mode 100644 FiftyOne.Pipeline.Engines.FiftyOne/Data/SetHeadersData.cs create mode 100644 FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/ISetHeadersElement.cs create mode 100644 FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs create mode 100644 FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElementBuilder.cs rename {Web Integration/FiftyOne.Pipeline.Web.Framework/Configuration => FiftyOne.Pipeline.Web.Shared}/PipelineWebIntegrationOptions.cs (66%) create mode 100644 Tests/FiftyOne.Pipeline.Engines.FiftyOne.Tests/FlowElements/SetHeadersElementTests.cs create mode 100644 Web Integration/FiftyOne.Pipeline.Web.Framework/Providers/SetHeadersProvider.cs delete mode 100644 Web Integration/FiftyOne.Pipeline.Web/PipelineWebIntegrationOptions.cs create mode 100644 Web Integration/FiftyOne.Pipeline.Web/Services/ISetHeadersService.cs create mode 100644 Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs diff --git a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs index cdd3339e..50bab81b 100644 --- a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs +++ b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs @@ -308,7 +308,8 @@ protected static void AddErrors(IFlowData data, if (data.Errors != null && data.Errors.Count > 0) { var errors = data.Errors - .Select(e => e.ExceptionData.Message); + .Select(e => e.ExceptionData.Message) + .ToArray(); allProperties.Add("errors", errors); } diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/Data/ISetHeadersData.cs b/FiftyOne.Pipeline.Engines.FiftyOne/Data/ISetHeadersData.cs new file mode 100644 index 00000000..ae570a94 --- /dev/null +++ b/FiftyOne.Pipeline.Engines.FiftyOne/Data/ISetHeadersData.cs @@ -0,0 +1,44 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.Data +{ + /// + /// The interface for + /// . + /// + public interface ISetHeadersData : IElementData + { + /// + /// A dictionary containing the names and values that engines + /// in the pipeline would like to be used to set HTTP headers + /// in the response. + /// + IReadOnlyDictionary ResponseHeaderDictionary { get; set; } + } +} diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/Data/SetHeadersData.cs b/FiftyOne.Pipeline.Engines.FiftyOne/Data/SetHeadersData.cs new file mode 100644 index 00000000..14195dbd --- /dev/null +++ b/FiftyOne.Pipeline.Engines.FiftyOne/Data/SetHeadersData.cs @@ -0,0 +1,69 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.Data +{ + /// + /// Element data instance for + /// + public class SetHeadersData : ElementDataBase, ISetHeadersData + { + /// + /// The key used to store the value for the + /// HTTP response headers in the internal data collection. + /// +#pragma warning disable CA1707 // Identifiers should not contain underscores + public const string RESPONSE_HEADERS_KEY = "responseheaderdictionary"; +#pragma warning restore CA1707 // Identifiers should not contain underscores + + /// + public SetHeadersData( + ILogger logger, + IPipeline pipeline) + : base(logger, pipeline) + { } + + /// + public SetHeadersData( + ILogger logger, + IPipeline pipeline, + IDictionary dictionary) + : base(logger, pipeline, dictionary) + { + } + + /// + public IReadOnlyDictionary ResponseHeaderDictionary + { + get => this[RESPONSE_HEADERS_KEY] as IReadOnlyDictionary; + set => this[RESPONSE_HEADERS_KEY] = value; + } + } +} diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/ISetHeadersElement.cs b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/ISetHeadersElement.cs new file mode 100644 index 00000000..662930ff --- /dev/null +++ b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/ISetHeadersElement.cs @@ -0,0 +1,41 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.FiftyOne.Data; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.FlowElements +{ + /// + /// An that collates responses from all + /// engines that want to set headers in the HTTP response in order + /// to gather additional data. + /// + public interface ISetHeadersElement : + IFlowElement + { + } +} diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs new file mode 100644 index 00000000..50c3c30b --- /dev/null +++ b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs @@ -0,0 +1,260 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.Data; +using FiftyOne.Pipeline.Engines.FiftyOne.Data; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.FlowElements +{ + /// + /// An that collates responses from all + /// engines that want to set headers in the HTTP response in order + /// to gather additional data. + /// + public class SetHeadersElement : + FlowElementBase, + ISetHeadersElement + { + /// + /// Contains configuration information relating to a particular + /// pipeline. + /// In most cases, a single instance of this element will only + /// be added to one pipeline at a time but it does support being + /// added to multiple pipelines simultaneously. + /// + protected class PipelineConfig + { + /// + /// A collection containing details about any properties + /// where the name starts with 'SetHeader' + /// + public Dictionary SetHeaderProperties { get; } = + new Dictionary (); + } + + /// + /// Used to store details of SetHeader* properties + /// + protected class PropertyDetails + { + /// + /// The property meta data + /// + public IElementPropertyMetaData PropertyMetaData { get; set; } + /// + /// The name of the HTTP header to set from the + /// value of this property. + /// + public string ResponseHeaderName { get; set; } + } + + private EvidenceKeyFilterWhitelist _evidenceKeyFilter; + private List _properties; + + private ConcurrentDictionary _pipelineConfigs; + + /// + public SetHeadersElement( + ILogger> logger, + Func, ISetHeadersData> elementDataFactory) + : base(logger, elementDataFactory) + { + _evidenceKeyFilter = + new EvidenceKeyFilterWhitelist(new List() { }); + _properties = new List() + { + new ElementPropertyMetaData( + this, "responseheaderdictionary", typeof(IReadOnlyDictionary), true) + }; + _pipelineConfigs = new ConcurrentDictionary(); + } + + /// + public override string ElementDataKey => "set-headers"; + + /// + public override IEvidenceKeyFilter EvidenceKeyFilter => _evidenceKeyFilter; + + /// + public override IList Properties => _properties; + + /// + protected override void ProcessInternal(IFlowData data) + { + if (data == null) throw new ArgumentNullException(nameof(data)); + + if (_pipelineConfigs.TryGetValue(data.Pipeline, out PipelineConfig config) == false) + { + config = PopulateConfig(data.Pipeline); + config = _pipelineConfigs.GetOrAdd(data.Pipeline, config); + } + + var elementData = data.GetOrAdd( + ElementDataKeyTyped, + CreateElementData); + var jsonString = BuildResponseHeaderDictionary(data, config); + elementData.ResponseHeaderDictionary = jsonString; + } + + /// + protected override void ManagedResourcesCleanup() + { + } + + /// + protected override void UnmanagedResourcesCleanup() + { + } + + private Dictionary BuildResponseHeaderDictionary( + IFlowData data, PipelineConfig config) + { + Dictionary result = new Dictionary(); + + // Iterate through 'SetHeader*' properties + foreach(var property in config.SetHeaderProperties) + { + // Get the value for this property. + var elementData = data.Get(property.Value.PropertyMetaData.Element.ElementDataKey); + var propertyValue = elementData[property.Key]; + // Extract the string value. + var headerValue = GetHeaderValue(propertyValue); + + // If value is not blank, null or 'unknown' then + // add it to the complete value for the associated + // header. + if(string.IsNullOrEmpty(headerValue) == false && + headerValue.Equals("Unknown", StringComparison.OrdinalIgnoreCase) == false) + { + if(result.TryGetValue(property.Value.ResponseHeaderName, + out string currentValue)) + { + // There is already an entry for this header name + // so concatenate the value. + result[property.Value.ResponseHeaderName] = + $"{currentValue},{headerValue}"; + } + else + { + // No entry for this header name so create it. + result.Add(property.Value.ResponseHeaderName, headerValue); + } + } + } + + return result; + } + + private static string GetHeaderValue(object propertyValue) + { + var result = string.Empty; + if (propertyValue is IAspectPropertyValue apv && + apv.HasValue) + { + result = apv.Value; + } + else if (propertyValue is string value) + { + result = value; + } + return result; + } + + /// + /// Executed on first request in order to build some collections + /// from the meta-data exposed by the Pipeline. + /// + private PipelineConfig PopulateConfig(IPipeline pipeline) + { + var config = new PipelineConfig(); + + // Populate the collection that contains a list of the + // properties with names starting with 'SetHeader' + foreach (var element in pipeline.ElementAvailableProperties) + { + foreach (var property in element.Value.Where(p => + p.Key.StartsWith("SetHeader", StringComparison.OrdinalIgnoreCase))) + { + PropertyDetails details = new PropertyDetails() + { + PropertyMetaData = property.Value, + ResponseHeaderName = GetResponseHeaderName(property.Key) + }; + config.SetHeaderProperties.Add(property.Key, details); + } + } + + return config; + } + + private static string GetResponseHeaderName(string propertyName) + { + if(propertyName.StartsWith("SetHeader", + StringComparison.Ordinal) == false) + { + throw new ArgumentException(string.Format( + CultureInfo.InvariantCulture, + Messages.ExceptionSetHeadersNotSetHeader, + propertyName), + nameof(propertyName)); + } + if(propertyName.Length < 11) + { + throw new ArgumentException(string.Format( + CultureInfo.InvariantCulture, + Messages.ExceptionSetHeadersWrongFormat, + propertyName), + nameof(propertyName)); + } + + int nextUpper = -1; + for(int i = 10; i < propertyName.Length; i++) + { + if (char.IsUpper(propertyName[i])) + { + nextUpper = i; + break; + } + } + + if(nextUpper == -1) + { + throw new ArgumentException(string.Format( + CultureInfo.InvariantCulture, + Messages.ExceptionSetHeadersWrongFormat, + propertyName), + nameof(propertyName)); + } + + return propertyName.Substring(nextUpper); + } + } +} diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElementBuilder.cs b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElementBuilder.cs new file mode 100644 index 00000000..bc4a28e2 --- /dev/null +++ b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElementBuilder.cs @@ -0,0 +1,88 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.Data; +using FiftyOne.Pipeline.Engines.FiftyOne.Data; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.FlowElements +{ + /// + /// Fluent builder for instances. + /// + public class SetHeadersElementBuilder + { + private ILoggerFactory _loggerFactory; + + /// + /// Constructor + /// + /// + /// The logger factory for this builder to use when creating new + /// instances. + /// + public SetHeadersElementBuilder(ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + } + + /// + /// Create a new instance. + /// + /// + public SetHeadersElement Build() + { + return new SetHeadersElement( + _loggerFactory.CreateLogger(), + CreateData); + } + + /// + /// Factory method for creating the + /// instances that + /// will be populated by the . + /// + /// + /// The pipeline that this is part of. + /// + /// + /// The the is creating this data + /// instance. + /// + /// + /// A new instance. + /// + private ISetHeadersData CreateData( + IPipeline pipeline, + FlowElementBase setHeadersElement) + { + return new SetHeadersData( + _loggerFactory.CreateLogger(), + pipeline); + } + } +} \ No newline at end of file diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/Messages.Designer.cs b/FiftyOne.Pipeline.Engines.FiftyOne/Messages.Designer.cs index 9683e594..0c6b24ef 100644 --- a/FiftyOne.Pipeline.Engines.FiftyOne/Messages.Designer.cs +++ b/FiftyOne.Pipeline.Engines.FiftyOne/Messages.Designer.cs @@ -60,6 +60,24 @@ internal Messages() { } } + /// + /// Looks up a localized string similar to Property name '{0}' does not start with 'SetHeader'. + /// + internal static string ExceptionSetHeadersNotSetHeader { + get { + return ResourceManager.GetString("ExceptionSetHeadersNotSetHeader", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Property name '{0}' is not in the expected format (SetHeader[Component][HeaderName]). + /// + internal static string ExceptionSetHeadersWrongFormat { + get { + return ResourceManager.GetString("ExceptionSetHeadersWrongFormat", resourceCulture); + } + } + /// /// Looks up a localized string similar to The minimum entries per message cannot be larger than the maximum size of the queue. /// diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/Messages.resx b/FiftyOne.Pipeline.Engines.FiftyOne/Messages.resx index 497f01fd..7ae24962 100644 --- a/FiftyOne.Pipeline.Engines.FiftyOne/Messages.resx +++ b/FiftyOne.Pipeline.Engines.FiftyOne/Messages.resx @@ -117,6 +117,12 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Property name '{0}' does not start with 'SetHeader' + + + Property name '{0}' is not in the expected format (SetHeader[Component][HeaderName]) + The minimum entries per message cannot be larger than the maximum size of the queue diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/Configuration/PipelineWebIntegrationOptions.cs b/FiftyOne.Pipeline.Web.Shared/PipelineWebIntegrationOptions.cs similarity index 66% rename from Web Integration/FiftyOne.Pipeline.Web.Framework/Configuration/PipelineWebIntegrationOptions.cs rename to FiftyOne.Pipeline.Web.Shared/PipelineWebIntegrationOptions.cs index 6f5c7b43..567d2899 100644 --- a/Web Integration/FiftyOne.Pipeline.Web.Framework/Configuration/PipelineWebIntegrationOptions.cs +++ b/FiftyOne.Pipeline.Web.Shared/PipelineWebIntegrationOptions.cs @@ -27,19 +27,44 @@ using System.Text; using System.Threading.Tasks; -namespace FiftyOne.Pipeline.Web.Framework.Configuration +namespace FiftyOne.Pipeline.Web.Shared { /// /// Extends the PipelineOptions class to add web specific options. /// public class PipelineWebIntegrationOptions : PipelineOptions - { + { + /// + /// Constructor + /// + public PipelineWebIntegrationOptions() + { + ClientSideEvidenceEnabled = true; + UseAsyncScript = true; + UseSetHeaderProperties = true; + } + /// /// True if client-side properties should be enabled. If enabled /// (and the JavaScriptBundlerElement added to the Pipeline), a /// client-side JavaScript file will be served at the URL /// */51Degrees.core.js. /// - public bool ClientSideEvidenceEnabled { get; set; } = true; + public bool ClientSideEvidenceEnabled { get; set; } + + /// + /// Flag to enable/disable the use of the async attribute for + /// the client side script. + /// Defaults to true. + /// + public bool UseAsyncScript { get; set; } + + /// + /// Flag to enable/disable a feature that will automatically set + /// the values of HTTP headers in the response in order to request + /// additional information. + /// Defaults to true. + /// + public bool UseSetHeaderProperties { get; set; } } } diff --git a/Tests/FiftyOne.Pipeline.Engines.FiftyOne.Tests/FlowElements/SetHeadersElementTests.cs b/Tests/FiftyOne.Pipeline.Engines.FiftyOne.Tests/FlowElements/SetHeadersElementTests.cs new file mode 100644 index 00000000..119bc773 --- /dev/null +++ b/Tests/FiftyOne.Pipeline.Engines.FiftyOne.Tests/FlowElements/SetHeadersElementTests.cs @@ -0,0 +1,275 @@ +using FiftyOne.Common.TestHelpers; +using FiftyOne.Pipeline.Engines.FlowElements; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Extensions.Logging; +using FiftyOne.Pipeline.Engines.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Engines.TestHelpers; +using Moq; +using Microsoft.Extensions.Primitives; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using FiftyOne.Pipeline.Engines.FiftyOne.Data; +using System.Linq; + +namespace FiftyOne.Pipeline.Engines.FiftyOne.Tests.FlowElements +{ + [TestClass] + public class SetHeadersElementTests + { + // These inner classes are used to create stubs for testing against. + #region inner classes + private class SetHeadersSourceData : ElementDataBase + { + public SetHeadersSourceData(ILogger logger, IPipeline pipeline) : base(logger, pipeline) + { + } + } + + private class ActivePropertySourceElement : FlowElementBase + { + private Dictionary _propertyNameValuesToReturn; + + public ActivePropertySourceElement( + ILogger> logger, + Func, SetHeadersSourceData> elementDataFactory, + Dictionary propertyNameValuesToReturn) + : base(logger, elementDataFactory) + { + _propertyNameValuesToReturn = propertyNameValuesToReturn; + } + + public override string ElementDataKey => "setheaderssourceelement"; + + public override IEvidenceKeyFilter EvidenceKeyFilter => new EvidenceKeyFilterWhitelist(new List()); + + public override IList Properties => _propertyNameValuesToReturn + .Select(p => new ElementPropertyMetaData(this, p.Key, typeof(object), true)) + .Cast() + .ToList(); + + protected override void ManagedResourcesCleanup() + { + } + + protected override void ProcessInternal(IFlowData data) + { + var sourceData = data.GetOrAdd(ElementDataKey, p => CreateElementData(p)); + sourceData.PopulateFromDictionary(_propertyNameValuesToReturn); + } + + protected override void UnmanagedResourcesCleanup() + { + } + } + #endregion + + private SetHeadersElement _element; + private ActivePropertySourceElement _sourceElement; + private TestLoggerFactory _loggerFactory; + private IPipeline _pipeline; + + [TestInitialize] + public void Init() + { + _loggerFactory = new TestLoggerFactory(); + } + + /// + /// Helper method to create the flow elements and configure the pipeline + /// + private void CreatePipeline( + Dictionary propertyNameValues) + { + _sourceElement = new ActivePropertySourceElement( + _loggerFactory.CreateLogger(), + (IPipeline pipeline, FlowElementBase element) => + { + return new SetHeadersSourceData(_loggerFactory.CreateLogger(), pipeline); + }, + propertyNameValues); + + _element = new SetHeadersElement( + _loggerFactory.CreateLogger(), + (IPipeline pipeline, FlowElementBase element) => + { + return new SetHeadersData(_loggerFactory.CreateLogger(), pipeline); + }); + + _pipeline = new PipelineBuilder(_loggerFactory) + .AddFlowElement(_sourceElement) + .AddFlowElement(_element) + .Build(); + } + + /// + /// The 'SetHeaderAcceptCH' property contains JSON for a single + /// header. + /// Output from SetHeadersElement should contain the expected + /// header value. + /// + [DataTestMethod] + // Output should be the same whether value is wrapped in + // AspectPropertyValue or not. + [DataRow(true)] + [DataRow(false)] + public void SetHeadersElement(bool valueIsAPV) + { + var valueStr = "UA-Platform"; + object value = valueStr; + if (valueIsAPV) + { + value = new AspectPropertyValue(valueStr); + } + + var propertyNameValues = new Dictionary() + { + { "SetHeaderBrowserAccept-CH", value } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + + // Verify the output + var typedOutput = GetFromFlowData(data); + Assert.AreEqual(1, typedOutput.ResponseHeaderDictionary.Count); + Assert.IsTrue(typedOutput.ResponseHeaderDictionary.ContainsKey("Accept-CH")); + Assert.AreEqual("UA-Platform", typedOutput.ResponseHeaderDictionary["Accept-CH"]); + } + + /// + /// The 'SetHeaderAcceptCH' property is set to various invalid values. + /// Output from SetHeadersElement should be an empty dictionary. + /// + [DataTestMethod] + [DataRow(null)] + [DataRow(123)] + [DataRow("Unknown")] + public void SetHeadersElement_InvalidPropertyValues(object sourcePropertyValue) + { + var propertyNameValues = new Dictionary() + { + { "SetHeaderBrowserAccept-CH", sourcePropertyValue } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + + // Verify the output + var typedOutput = GetFromFlowData(data); + Assert.AreEqual(0, typedOutput.ResponseHeaderDictionary.Count); + } + + /// + /// The 'SetHeaderAcceptCH' property is set to various invalid values + /// that are wrapped in an AspectPropertyValue. + /// Output from SetHeadersElement should be an empty dictionary. + /// + [DataTestMethod] + [DataRow(false)] + [DataRow(true, null)] + public void SetHeadersElement_APV_InvalidPropertyValues(bool hasValue, string sourcePropertyValue = null) + { + var value = new AspectPropertyValue(); + if (hasValue) + { + value.Value = sourcePropertyValue; + } + + var propertyNameValues = new Dictionary() + { + { "SetHeaderBrowserAccept-CH", value } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + + // Verify the output + var typedOutput = GetFromFlowData(data); + Assert.AreEqual(0, typedOutput.ResponseHeaderDictionary.Count); + } + + /// + /// Test that various invalid property names cause + /// an exception to be thrown + /// + [DataTestMethod] + [DataRow("SetHeader")] + [DataRow("SetHeaderBrowser")] + [ExpectedException(typeof(AggregateException))] + public void SetHeadersElement_InvalidPropertyNames(string sourcePropertyName) + { + var propertyNameValues = new Dictionary() + { + { sourcePropertyName, "TEST" } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + } + + + /// + /// Test that multiple properties will result in multiple headers + /// being set in the response. + /// + [TestMethod] + public void SetHeadersElement_MultipleProperties() + { + var propertyNameValues = new Dictionary() + { + { "SetHeaderBrowserAccept-CH", "Sec-CH-UA" }, + { "SetHeaderHardwareCritical-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile" } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + + var typedOutput = GetFromFlowData(data); + Assert.AreEqual(2, typedOutput.ResponseHeaderDictionary.Count); + Assert.IsTrue(typedOutput.ResponseHeaderDictionary.ContainsKey("Accept-CH")); + Assert.IsTrue(typedOutput.ResponseHeaderDictionary.ContainsKey("Critical-CH")); + Assert.AreEqual("Sec-CH-UA", + typedOutput.ResponseHeaderDictionary["Accept-CH"]); + Assert.AreEqual("Sec-CH-UA-Model,Sec-CH-UA-Mobile", + typedOutput.ResponseHeaderDictionary["Critical-CH"]); + } + + /// + /// Test that the SetHeadersElement will combine values from + /// multiple properties that are associated with the same header. + /// + [TestMethod] + public void SetHeadersElement_MultipleProperties_SameHeader() + { + var propertyNameValues = new Dictionary() + { + { "SetHeaderBrowserAccept-CH", "Sec-CH-UA" }, + { "SetHeaderHardwareAccept-CH", "Sec-CH-UA-Model,Sec-CH-UA-Mobile" } + }; + CreatePipeline(propertyNameValues); + var data = _pipeline.CreateFlowData(); + data.Process(); + + var typedOutput = GetFromFlowData(data); + Assert.AreEqual(1, typedOutput.ResponseHeaderDictionary.Count); + Assert.IsTrue(typedOutput.ResponseHeaderDictionary.ContainsKey("Accept-CH")); + Assert.AreEqual("Sec-CH-UA,Sec-CH-UA-Model,Sec-CH-UA-Mobile", + typedOutput.ResponseHeaderDictionary["Accept-CH"]); + } + + private SetHeadersData GetFromFlowData(IFlowData data) + { + var output = data.ElementDataAsDictionary(); + var elementOutput = output[_element.ElementDataKey]; + Assert.IsNotNull(elementOutput); + Assert.IsInstanceOfType(elementOutput, typeof(SetHeadersData)); + var typedOutput = elementOutput as SetHeadersData; + Assert.IsNotNull(typedOutput.ResponseHeaderDictionary); + return typedOutput; + } + } +} diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/FiftyOne.Pipeline.Web.Framework.csproj b/Web Integration/FiftyOne.Pipeline.Web.Framework/FiftyOne.Pipeline.Web.Framework.csproj index 2f1a4c1c..5cc7417f 100644 --- a/Web Integration/FiftyOne.Pipeline.Web.Framework/FiftyOne.Pipeline.Web.Framework.csproj +++ b/Web Integration/FiftyOne.Pipeline.Web.Framework/FiftyOne.Pipeline.Web.Framework.csproj @@ -63,13 +63,13 @@ - + diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/Providers/SetHeadersProvider.cs b/Web Integration/FiftyOne.Pipeline.Web.Framework/Providers/SetHeadersProvider.cs new file mode 100644 index 00000000..f037744a --- /dev/null +++ b/Web Integration/FiftyOne.Pipeline.Web.Framework/Providers/SetHeadersProvider.cs @@ -0,0 +1,136 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Web; + +namespace FiftyOne.Pipeline.Web.Framework.Providers +{ + /// + /// This class handles setting HTTP headers in the response based + /// on values from the + /// + public class SetHeadersProvider + { + /// + /// The single instance of the provider. + /// + private static SetHeadersProvider _instance = null; + + /// + /// Lock used when constructing the instance. + /// + private static readonly object _lock = new object(); + + /// + /// The set headers flow element that generates the dictionary + /// of response header values. + /// + private ISetHeadersElement _setHeadersElement; + + /// + /// Get the single instance of the provider. If one does not yet + /// exist, it is constructed. + /// + /// + public static SetHeadersProvider GetInstance() + { + if (_instance == null) + { + lock (_lock) + { + if (_instance == null) + { + _instance = new SetHeadersProvider(); + } + } + } + return _instance; + } + + /// + /// Constructor + /// + public SetHeadersProvider() + { + var pipeline = WebPipeline.GetInstance().Pipeline; + _setHeadersElement = pipeline.GetElement(); + } + + /// + /// Set the HTTP headers in the response based + /// on values from the + /// + /// + /// The HTTP context + /// + public void SetHeaders(HttpContext context) + { + if (context == null) throw new ArgumentNullException(nameof(context)); + + PipelineCapabilities caps = context.Request.Browser as PipelineCapabilities; + var flowData = caps.FlowData; + + if (_setHeadersElement != null) + { + var headersToSet = flowData + .GetFromElement(_setHeadersElement).ResponseHeaderDictionary; + SetHeaders(context, headersToSet); + } + } + + /// + /// Set the HTTP headers in the response based + /// on values in the supplied headersToSet parameter. + /// + /// + /// The HTTP context + /// + /// + /// A dictionary containing the names and values of the headers + /// to set. + /// + public static void SetHeaders(HttpContext context, + IReadOnlyDictionary headersToSet) + { + if (context == null) throw new ArgumentNullException(nameof(context)); + if (headersToSet == null) throw new ArgumentNullException(nameof(headersToSet)); + + foreach (var header in headersToSet) + { + if (context.Response.Headers.AllKeys.Contains(header.Key)) + { + context.Response.Headers[header.Key] += $",{header.Value}"; + } + else + { + context.Response.Headers.Add(header.Key, header.Value); + } + } + } + } +} \ No newline at end of file diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs index e7c1d202..35031cd1 100644 --- a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs +++ b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs @@ -28,6 +28,7 @@ using FiftyOne.Pipeline.JavaScriptBuilder.FlowElement; using FiftyOne.Pipeline.JsonBuilder.FlowElement; using FiftyOne.Pipeline.Web.Framework.Configuration; +using FiftyOne.Pipeline.Web.Framework.Providers; using FiftyOne.Pipeline.Web.Shared; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -55,6 +56,11 @@ public class WebPipeline /// public bool ClientSideEvidenceEnabled => _options.ClientSideEvidenceEnabled; + /// + /// Whether or not set header properties are enabled. + /// + public bool SetHeaderPropertiesEnabled => _options.UseSetHeaderProperties; + /// /// Extra pipeline options which only apply to an implementation in a /// web server. @@ -237,6 +243,14 @@ public static IFlowData Process(HttpRequest request) // Process the evidence and return the result flowData.Process(); + + if (GetInstance().SetHeaderPropertiesEnabled) + { + // Set HTTP headers in the response. + SetHeadersProvider.GetInstance().SetHeaders( + request.RequestContext.HttpContext.ApplicationInstance.Context); + } + return flowData; } diff --git a/Web Integration/FiftyOne.Pipeline.Web/Constants.cs b/Web Integration/FiftyOne.Pipeline.Web/Constants.cs index bcfaa0ed..8cf3780b 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/Constants.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/Constants.cs @@ -57,5 +57,31 @@ public static class Constants internal const string ClientSidePropertyCopyright = "// Copyright 51Degrees Mobile Experts Limited"; + /// + /// Element datakey to get response set header properties. + /// + public const string ELEMENT_DATAKEY = "device"; + + /// + /// UACH response header name. + /// + public const string ACCEPTCH_HEADER = "Accept-CH"; + + /// + /// UACH SetHeaderBrowserAccept-CH property key value. + /// + + public const string ACCEPTCH_BROWSER = "setheaderbrowseraccept-ch"; + + /// + /// UACH SetHeaderPlatformAccept-CH property key value. + /// + public const string ACCEPTCH_PLATFORM = "setheaderplatformaccept-ch"; + + /// + /// UACH SetHeaderHardwareAccept-CH property key value. + /// + public const string ACCEPTCH_HARDWARE = "setheaderhardwareaccept-ch"; + } } diff --git a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneJSViewComponent.cs b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneJSViewComponent.cs index a6a35ce5..b8c8ca96 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneJSViewComponent.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneJSViewComponent.cs @@ -20,6 +20,7 @@ * such notice(s) shall fulfill the requirements of that article. * ********************************************************************* */ +using FiftyOne.Pipeline.Web.Shared; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; diff --git a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneMiddleware.cs b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneMiddleware.cs index 992523b4..8517d078 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneMiddleware.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneMiddleware.cs @@ -25,6 +25,7 @@ using FiftyOne.Pipeline.Web.Services; using FiftyOne.Pipeline.Core.Data; using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; namespace FiftyOne.Pipeline.Web { @@ -51,6 +52,17 @@ public class FiftyOneMiddleware /// accessible through the . /// protected IPipelineResultService PipelineResultService { get; private set; } + /// + /// A service to get FlowData Object from response + /// + protected IFlowDataProvider FlowDataProvider { get; private set; } + + /// + /// The SetHeaderService sets the values of headers in the + /// response in order to request relevant information from + /// the client. + /// + protected ISetHeadersService HeaderService { get; private set; } /// /// Create a new FiftyOneMiddleware object. @@ -66,13 +78,24 @@ public class FiftyOneMiddleware /// /// A service that can serve the 51Degrees JavaScript if needed /// + /// + /// A service to get FlowData Object from response + /// + /// + /// A service that can set headers in the response based on + /// data from an . + /// public FiftyOneMiddleware(RequestDelegate next, IPipelineResultService pipelineResultService, - IFiftyOneJSService jsService) + IFiftyOneJSService jsService, + IFlowDataProvider flowDataProvider, + ISetHeadersService headerService) { Next = next; PipelineResultService = pipelineResultService; JsService = jsService; + FlowDataProvider = flowDataProvider; + HeaderService = headerService; } /// @@ -95,7 +118,9 @@ public async Task Invoke(HttpContext context) // Populate the request properties and store against the // HttpContext. PipelineResultService.Process(context); - + // Set HTTP headers in the response based on the results + // from the engines in the pipeline. + HeaderService.SetHeaders(context); // If 51Degrees JavaScript or JSON is being requested then serve it. // Otherwise continue down the middleware Pipeline. if (JsService.ServeJS(context) == false && diff --git a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs index 910887b0..f271dd6d 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs @@ -63,7 +63,7 @@ public static class FiftyOneStartup /// The function used to create the pipeline. If null is passed /// then this will default to the CreatePipelineFromConfig method /// - internal static void ConfigureServices(IServiceCollection services, + internal static void ConfigureServices(IServiceCollection services, IConfiguration configuration, Func pipelineFactory) where TBuilder : class, IPipelineBuilderFromConfiguration @@ -99,7 +99,8 @@ internal static void ConfigureServices(IServiceCollection services, services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(IServiceCollection services, services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Add the pipeline to the DI container services.AddSingleton(serviceProvider => { IPipeline pipeline = null; // Create the pipeline builder - var pipelineBuilder = + var pipelineBuilder = serviceProvider.GetRequiredService(); if (pipelineFactory == null) @@ -135,7 +137,7 @@ internal static void ConfigureServices(IServiceCollection services, return pipeline; }); } - + /// /// The default factory function. /// This looks for a 'PipelineOptions' configuration item and uses @@ -171,77 +173,119 @@ private static IPipeline CreatePipelineFromConfig( config.Bind("PipelineWebIntegrationOptions", webOptions); // Add the sequence element. - var sequenceConfig = options.Elements.Where(e => - e.BuilderName.Contains(nameof(SequenceElement), - StringComparison.OrdinalIgnoreCase)); - if (sequenceConfig.Any() == false) - { - // The sequence element is not included so add it. - // Make sure it's added as the first element. - options.Elements.Insert(0, new ElementOptions() - { - BuilderName = nameof(SequenceElement) - }); - } + AddSequenceElement(options); if (webOptions.ClientSideEvidenceEnabled) { // Client-side evidence is enabled so make sure the // JsonBuilderElement and JavaScriptBundlerElement has been // included. - var jsonConfig = options.Elements.Where(e => - e.BuilderName.Contains(nameof(JsonBuilderElement), - StringComparison.OrdinalIgnoreCase)); - var javascriptConfig = options.Elements.Where(e => - e.BuilderName.Contains(nameof(JavaScriptBuilderElement), - StringComparison.OrdinalIgnoreCase)); + AddJsElements(options); + } - var jsIndex = javascriptConfig.Any() ? - options.Elements.IndexOf(javascriptConfig.First()) : -1; + // Add the SetHeaders element + AddSetHeadersElement(options); - if (jsonConfig.Any() == false) - { - // The json builder is not included so add it. - var newElementOptions = new ElementOptions() - { - BuilderName = nameof(JsonBuilderElement) - }; - if (jsIndex > -1) - { - // There is already a javascript builder element - // so insert the json builder before it. - options.Elements.Insert(jsIndex, newElementOptions); - } - else - { - options.Elements.Add(newElementOptions); - } - } + return pipelineBuilder.BuildFromConfiguration(options); + } - if (jsIndex == -1) + /// + /// Ensure the json and javascript elements are added to the configuration + /// + /// + private static void AddJsElements(PipelineOptions options) + { + var jsonConfig = options.Elements.Where(e => + e.BuilderName.Contains(nameof(JsonBuilderElement), + StringComparison.OrdinalIgnoreCase)); + var javascriptConfig = options.Elements.Where(e => + e.BuilderName.Contains(nameof(JavaScriptBuilderElement), + StringComparison.OrdinalIgnoreCase)); + + var jsIndex = javascriptConfig.Any() ? + options.Elements.IndexOf(javascriptConfig.First()) : -1; + + if (jsonConfig.Any() == false) + { + // The json builder is not included so add it. + var newElementOptions = new ElementOptions() { - // The builder is not included so add it. - options.Elements.Add(new ElementOptions() - { - BuilderName = nameof(JavaScriptBuilderElement), - BuildParameters = new Dictionary() - { - { "EndPoint", "/51dpipeline/json" } - } - }); + BuilderName = nameof(JsonBuilderElement) + }; + if (jsIndex > -1) + { + // There is already a javascript builder element + // so insert the json builder before it. + options.Elements.Insert(jsIndex, newElementOptions); } else { - // There is already a JavaScript builder config so check if - // the endpoint is specified. If not, add it. - if (javascriptConfig.Single().BuildParameters.ContainsKey("EndPoint") == false) + options.Elements.Add(newElementOptions); + } + } + + if (jsIndex == -1) + { + // The builder is not included so add it. + options.Elements.Add(new ElementOptions() + { + BuilderName = nameof(JavaScriptBuilderElement), + BuildParameters = new Dictionary() { - javascriptConfig.Single().BuildParameters.Add("EndPoint", "/51dpipeline/json"); + { "EndPoint", "/51dpipeline/json" } } + }); + } + else + { + // There is already a JavaScript builder config so check if + // the endpoint is specified. If not, add it. + if (jsonConfig.Single().BuildParameters.ContainsKey("EndPoint") == false) + { + jsonConfig.Single().BuildParameters.Add("EndPoint", "/51dpipeline/json"); } } + } - return pipelineBuilder.BuildFromConfiguration(options); + /// + /// Ensure the sequence element is added to the configuration + /// + /// + private static void AddSequenceElement(PipelineOptions options) + { + var sequenceConfig = options.Elements.Where(e => + e.BuilderName.Contains(nameof(SequenceElement), + StringComparison.OrdinalIgnoreCase)); + if (sequenceConfig.Any() == false) + { + // The sequence element is not included so add it. + // Make sure it's added as the first element. + options.Elements.Insert(0, new ElementOptions() + { + BuilderName = nameof(SequenceElement) + }); + } + } + + /// + /// Ensure the set headers element is added to the configuration + /// + /// + private static void AddSetHeadersElement(PipelineOptions options) + { + var setHeadersConfig = options.Elements.Where(e => + e.BuilderName.Contains(nameof(SetHeadersElement), + StringComparison.OrdinalIgnoreCase)); + if (setHeadersConfig.Any() == false) + { + // The set headers element is not included, so add it. + // Make sure it's added as the last element. + options.Elements.Add(new ElementOptions() + { + BuilderName = nameof(SetHeadersElement) + }); + } } } } + diff --git a/Web Integration/FiftyOne.Pipeline.Web/PipelineWebIntegrationOptions.cs b/Web Integration/FiftyOne.Pipeline.Web/PipelineWebIntegrationOptions.cs deleted file mode 100644 index b7ae801c..00000000 --- a/Web Integration/FiftyOne.Pipeline.Web/PipelineWebIntegrationOptions.cs +++ /dev/null @@ -1,59 +0,0 @@ -/* ********************************************************************* - * This Original Work is copyright of 51 Degrees Mobile Experts Limited. - * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, - * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. - * - * This Original Work is licensed under the European Union Public Licence (EUPL) - * v.1.2 and is subject to its terms as set out below. - * - * If a copy of the EUPL was not distributed with this file, You can obtain - * one at https://opensource.org/licenses/EUPL-1.2. - * - * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be - * amended by the European Commission) shall be deemed incompatible for - * the purposes of the Work and the provisions of the compatibility - * clause in Article 5 of the EUPL shall not apply. - * - * If using the Work as, or as part of, a network application, by - * including the attribution notice(s) required under Article 5 of the EUPL - * in the end user terms of the application under an appropriate heading, - * such notice(s) shall fulfill the requirements of that article. - * ********************************************************************* */ - -namespace FiftyOne.Pipeline.Web -{ - /// - /// Configuration options for MVC Pipeline operation. - /// - public class PipelineWebIntegrationOptions - { - /// - /// Constructor. - /// - public PipelineWebIntegrationOptions() - { - ClientSideEvidenceEnabled = true; - UseAsyncScript = true; - } - - /// - /// Flag to enable/disable client side evidence functionality. - /// Client-side evidence comes into effect when there is not enough - /// information in the request to determine certain properties. - /// For example the exact model of iPhone cannot be determined - /// from the User-Agent. - /// If enabled, this option allows the Pipeline to inject JavaScript - /// into the page and use this to determine a value for a property - /// using more information. - /// Defaults to true. - /// - public bool ClientSideEvidenceEnabled { get; set; } - - /// - /// Flag to enable/disable the use of the async attribute for - /// the client side script. - /// Defaults to true. - /// - public bool UseAsyncScript { get; set; } - } -} diff --git a/Web Integration/FiftyOne.Pipeline.Web/Services/FiftyOneJSService.cs b/Web Integration/FiftyOne.Pipeline.Web/Services/FiftyOneJSService.cs index 0b30a735..545acb06 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/Services/FiftyOneJSService.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/Services/FiftyOneJSService.cs @@ -22,6 +22,7 @@ using FiftyOne.Pipeline.Core.Data; using FiftyOne.Pipeline.Web.Adapters; +using FiftyOne.Pipeline.Web.Shared; using FiftyOne.Pipeline.Web.Shared.Services; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Options; diff --git a/Web Integration/FiftyOne.Pipeline.Web/Services/ISetHeadersService.cs b/Web Integration/FiftyOne.Pipeline.Web/Services/ISetHeadersService.cs new file mode 100644 index 00000000..067156c4 --- /dev/null +++ b/Web Integration/FiftyOne.Pipeline.Web/Services/ISetHeadersService.cs @@ -0,0 +1,49 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using Microsoft.AspNetCore.Http; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Web.Services +{ + /// + /// This service handles setting HTTP headers in the response for + /// properties in the results from the pipeline that start with + /// 'SetHeader' + /// + public interface ISetHeadersService + { + /// + /// Check if there are any populated SetHeader* properties. + /// if there are then use the data they contain to populate + /// the specified headers in the response. + /// + /// + /// The HttpContext + /// + /// + /// + void SetHeaders(HttpContext context); + } +} \ No newline at end of file diff --git a/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs b/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs new file mode 100644 index 00000000..e6339e06 --- /dev/null +++ b/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs @@ -0,0 +1,132 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2020 51 Degrees Mobile Experts Limited, 5 Charlotte Close, + * Caversham, Reading, Berkshire, United Kingdom RG4 7BY. + * + * This Original Work is licensed under the European Union Public Licence (EUPL) + * v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.Data; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using FiftyOne.Pipeline.Engines.FlowElements; +using FiftyOne.Pipeline.Web.Shared; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Web.Services +{ + /// + public class SetHeaderService : ISetHeadersService + { + private ILogger _logger; + + /// + /// Data provider + /// + private IFlowDataProvider _flowDataProvider; + + /// + /// The active property flow element that generates the dictionary + /// of response header values. + /// + private ISetHeadersElement _setHeadersElement; + + private IOptions _options; + + /// + /// Constructor + /// + /// + /// + /// + /// + public SetHeaderService( + ILogger logger, + IPipeline pipeline, + IFlowDataProvider flowDataProvider, + IOptions options) + { + if (logger == null) throw new ArgumentNullException(nameof(logger)); + if (pipeline == null) throw new ArgumentNullException(nameof(pipeline)); + if (flowDataProvider == null) throw new ArgumentNullException(nameof(flowDataProvider)); + + _logger = logger; + _flowDataProvider = flowDataProvider; + _setHeadersElement = pipeline.GetElement(); + _options = options; + } + + /// + public void SetHeaders(HttpContext context) + { + if (_setHeadersElement != null && + _options.Value.UseSetHeaderProperties) + { + var headersToSet = _flowDataProvider.GetFlowData() + .GetFromElement(_setHeadersElement).ResponseHeaderDictionary; + if (_logger.IsEnabled(LogLevel.Debug)) + { + foreach (var header in headersToSet) + { + _logger.LogDebug($"Adding response header " + + $"'{header.Key}' with value '{header.Value}'"); + } + } + SetHeaders(context, headersToSet); + } + } + + /// + /// Set the HTTP headers in the response using the supplied + /// dictionary. + /// If the supplied headers already have values in the response + /// then they will be amended rather than replaced. + /// + /// + /// The to set the response headers in + /// + /// + /// A dictionary containing the names and values of the headers + /// to set. + /// + public static void SetHeaders(HttpContext context, + IReadOnlyDictionary headersToSet) + { + if (context == null) throw new ArgumentNullException(nameof(context)); + if (headersToSet == null) throw new ArgumentNullException(nameof(headersToSet)); + + foreach (var header in headersToSet) + { + if (context.Response.Headers.ContainsKey(header.Key)) + { + context.Response.Headers.Append(header.Key, header.Value); + } + else + { + context.Response.Headers.Add(header.Key, header.Value); + } + } + } + } +} \ No newline at end of file diff --git a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOne.Pipeline.Web.Tests.csproj b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOne.Pipeline.Web.Tests.csproj index e9181410..c8f09c44 100644 --- a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOne.Pipeline.Web.Tests.csproj +++ b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOne.Pipeline.Web.Tests.csproj @@ -13,6 +13,8 @@ + + diff --git a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOneMiddlewareTest.cs b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOneMiddlewareTest.cs index 4bf9f402..10b2dd79 100644 --- a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOneMiddlewareTest.cs +++ b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/FiftyOneMiddlewareTest.cs @@ -38,6 +38,10 @@ public class FiftyOneMiddlewareTest private Mock _jsService; + private Mock _flowDataProvider; + + private Mock _setHeadersService; + private bool _movedNext; /// @@ -50,7 +54,9 @@ public void SetUp() { _movedNext = false; _resultsService = new Mock(); - _jsService = new Mock(); + _jsService = new Mock(); + _flowDataProvider = new Mock(); + _setHeadersService = new Mock(); _middleware = new FiftyOneMiddleware( delegate (HttpContext context) { @@ -58,7 +64,9 @@ public void SetUp() return Task.FromResult(null); }, _resultsService.Object, - _jsService.Object); + _jsService.Object, + _flowDataProvider.Object, + _setHeadersService.Object); } /// @@ -78,6 +86,12 @@ public void FiftyOneMiddleware_Invoke() s => s.Process(context), Times.Once, "The results were not processed."); + + _setHeadersService.Verify( + s => s.SetHeaders(context), + Times.Once, + "The response headers were not set as expected"); + Assert.IsTrue(_movedNext, "The next middleware was not called."); } @@ -99,6 +113,12 @@ public void FiftyOneMiddleware_InvokeJs() s => s.Process(It.IsAny()), Times.Once, "The results were not processed."); + + _setHeadersService.Verify( + s => s.SetHeaders(context), + Times.Once, + "The response headers were not set as expected"); + Assert.IsFalse( _movedNext, "The next middleware should not have been called."); From d22e47b6542f0f7a8c04382ed4679bb11dc52733 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Fri, 23 Apr 2021 15:22:33 +0100 Subject: [PATCH 02/14] Update build pipeline to use version 5 of GitVersion --- publish-packages.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/publish-packages.yml b/publish-packages.yml index 6ca1d935..faa2e173 100644 --- a/publish-packages.yml +++ b/publish-packages.yml @@ -85,12 +85,13 @@ steps: feedsToUse: 'select' vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' -- task: gittools.gitversion.gitversion-task.GitVersion@4 +- task: UseGitVersion@5 displayName: 'Determine Version Number' # Give this task a name so we can use the variables it sets later. name: GitVersion inputs: - preferBundledVersion: false + versionSpec: '5.x' + updateAssemblyInfo: true - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' From efb10957e3456205ddc8f1703832a7f94aa31495 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Fri, 23 Apr 2021 15:32:00 +0100 Subject: [PATCH 03/14] BUILD: Update to newer syntax for gitversion --- publish-packages.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/publish-packages.yml b/publish-packages.yml index faa2e173..3dc1cdfe 100644 --- a/publish-packages.yml +++ b/publish-packages.yml @@ -84,14 +84,14 @@ steps: restoreSolution: '$(RestoreBuildProjects)' feedsToUse: 'select' vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' - -- task: UseGitVersion@5 - displayName: 'Determine Version Number' - # Give this task a name so we can use the variables it sets later. - name: GitVersion + +- task: gitversion/setup@0 + displayName: Install GitVersion inputs: versionSpec: '5.x' - updateAssemblyInfo: true + +- task: gitversion/execute@0 + displayName: Determine Version - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' @@ -118,15 +118,13 @@ steps: SymbolServerType: 'TeamServices' SymbolsVersion: '$(GitVersion.NuGetVersion)' -# The nuget package version uses the BUILD_BUILDNUMER environment variable. -# This has been set by the GitVersion task above. - task: DotNetCoreCLI@2 displayName: 'Build NuGet Package' inputs: command: 'pack' packagesToPack: '$(PublishProjects)' versioningScheme: 'byEnvVar' - versionEnvVar: 'BUILD_BUILDNUMBER' + versionEnvVar: GitVersion.NuGetVersion # The Web and Web.Framework projects are combined into a single NuGet package. # This requires the use of a nuspec file and the NuGet task. @@ -136,7 +134,7 @@ steps: command: 'pack' packagesToPack: '**/FiftyOne.Pipeline.Web.nuspec' versioningScheme: 'byEnvVar' - versionEnvVar: 'BUILD_BUILDNUMBER' + versionEnvVar: GitVersion.NuGetVersion buildProperties: 'config=$(BuildConfiguration)' # The secure file to download will be stored in the From e926b06bfa1c6558a874b64e001df7c6cf307483 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Fri, 23 Apr 2021 16:24:22 +0100 Subject: [PATCH 04/14] BUILD: Use the BUILD_NUMBER environment variable to set nuget package version, rather than GitVersion.NuGetVersion. --- publish-packages.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/publish-packages.yml b/publish-packages.yml index 3dc1cdfe..59fd4c31 100644 --- a/publish-packages.yml +++ b/publish-packages.yml @@ -118,13 +118,15 @@ steps: SymbolServerType: 'TeamServices' SymbolsVersion: '$(GitVersion.NuGetVersion)' +# The nuget package version uses the BUILD_BUILDNUMER environment variable. +# This has been set by the GitVersion task above. - task: DotNetCoreCLI@2 displayName: 'Build NuGet Package' inputs: command: 'pack' packagesToPack: '$(PublishProjects)' versioningScheme: 'byEnvVar' - versionEnvVar: GitVersion.NuGetVersion + versionEnvVar: 'BUILD_BUILDNUMBER' # The Web and Web.Framework projects are combined into a single NuGet package. # This requires the use of a nuspec file and the NuGet task. @@ -134,7 +136,7 @@ steps: command: 'pack' packagesToPack: '**/FiftyOne.Pipeline.Web.nuspec' versioningScheme: 'byEnvVar' - versionEnvVar: GitVersion.NuGetVersion + versionEnvVar: 'BUILD_BUILDNUMBER' buildProperties: 'config=$(BuildConfiguration)' # The secure file to download will be stored in the From ddbfe525cfd7beb8df17c0b827a041d95e17f96f Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Fri, 23 Apr 2021 17:05:55 +0000 Subject: [PATCH 05/14] REFACTOR: Modify SetHeadersService.SetHeaders to take a IFlowData instead of a dictionary. REFACTOR: Modify SetHeadersService.SetHeaders to take a IFlowData instead of a dictionary. --- .../Services/SetHeadersService.cs | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs b/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs index e6339e06..be084551 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/Services/SetHeadersService.cs @@ -20,6 +20,7 @@ * such notice(s) shall fulfill the requirements of that article. * ********************************************************************* */ +using FiftyOne.Pipeline.Core.Data; using FiftyOne.Pipeline.Core.FlowElements; using FiftyOne.Pipeline.Engines.Data; using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; @@ -46,12 +47,6 @@ public class SetHeaderService : ISetHeadersService /// private IFlowDataProvider _flowDataProvider; - /// - /// The active property flow element that generates the dictionary - /// of response header values. - /// - private ISetHeadersElement _setHeadersElement; - private IOptions _options; /// @@ -73,50 +68,39 @@ public SetHeaderService( _logger = logger; _flowDataProvider = flowDataProvider; - _setHeadersElement = pipeline.GetElement(); _options = options; } /// public void SetHeaders(HttpContext context) { - if (_setHeadersElement != null && - _options.Value.UseSetHeaderProperties) + if (_options.Value.UseSetHeaderProperties) { - var headersToSet = _flowDataProvider.GetFlowData() - .GetFromElement(_setHeadersElement).ResponseHeaderDictionary; - if (_logger.IsEnabled(LogLevel.Debug)) - { - foreach (var header in headersToSet) - { - _logger.LogDebug($"Adding response header " + - $"'{header.Key}' with value '{header.Value}'"); - } - } - SetHeaders(context, headersToSet); + SetHeaders(context, _flowDataProvider.GetFlowData()); } } /// - /// Set the HTTP headers in the response using the supplied - /// dictionary. + /// Set the HTTP headers in the response using values from + /// the supplied flow data. /// If the supplied headers already have values in the response /// then they will be amended rather than replaced. /// /// /// The to set the response headers in /// - /// - /// A dictionary containing the names and values of the headers - /// to set. + /// + /// The flow data containing the headers to set. /// public static void SetHeaders(HttpContext context, - IReadOnlyDictionary headersToSet) + IFlowData flowData) { if (context == null) throw new ArgumentNullException(nameof(context)); - if (headersToSet == null) throw new ArgumentNullException(nameof(headersToSet)); + if (flowData == null) throw new ArgumentNullException(nameof(flowData)); - foreach (var header in headersToSet) + var element = flowData.Pipeline.GetElement(); + foreach (var header in flowData.GetFromElement(element) + .ResponseHeaderDictionary) { if (context.Response.Headers.ContainsKey(header.Key)) { From a26f364526065c252a9feb87560c33ebcbb4ea37 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Wed, 28 Apr 2021 09:10:40 +0000 Subject: [PATCH 06/14] FEAT: Exclude SetHeaderElement from the json builder. FEAT: Exclude SetHeaderElement from the json builder. CLEANUP: Removed most references to 'blacklist' or 'whitelist' from the code and comments. Some references remain in the public interface classes and methods. --- .../Data/EvidenceKeyFilterAggregator.cs | 20 +++---- .../Data/EvidenceKeyFilterWhitelist.cs | 54 +++++++++---------- .../Data/IEvidenceKeyFilter.cs | 2 +- .../FlowElement/JsonBuilderElement.cs | 30 +++++++---- .../JsonBuilderElementTests.cs | 8 +-- .../Data/EvidenceKeyFilterShareUsage.cs | 2 +- .../FlowElements/SetHeadersElement.cs | 9 +++- .../EmptyEngine.cs | 4 +- .../Services/IDataUpdateService.cs | 2 +- .../Services/ClientsidePropertyService.cs | 6 +-- 10 files changed, 78 insertions(+), 59 deletions(-) diff --git a/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterAggregator.cs b/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterAggregator.cs index c27fc5a5..37cb5643 100644 --- a/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterAggregator.cs +++ b/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterAggregator.cs @@ -54,22 +54,22 @@ public EvidenceKeyFilterAggregator() : /// public void AddFilter(IEvidenceKeyFilter filter) { - var whiteListFilter = filter as EvidenceKeyFilterWhitelist; + var inclusionListFilter = filter as EvidenceKeyFilterWhitelist; bool addFilter = true; - if (whiteListFilter != null) + if (inclusionListFilter != null) { - // If the filter is a white list filter using the OrdinalIgnoreCase - // comparer then add it's list to this instance's white list to - // give better performance. - if (whiteListFilter.Comparer == StringComparer.OrdinalIgnoreCase) + // If the filter is an inclusion list filter using the + // OrdinalIgnoreCase comparer then add it's list to this + // instance's inclusion list to give better performance. + if (inclusionListFilter.Comparer == StringComparer.OrdinalIgnoreCase) { addFilter = false; - foreach (var entry in whiteListFilter.Whitelist) + foreach (var entry in inclusionListFilter.Whitelist) { - if (_whitelist.ContainsKey(entry.Key) == false) + if (_inclusionList.ContainsKey(entry.Key) == false) { - _whitelist.Add(entry.Key, entry.Value); + _inclusionList.Add(entry.Key, entry.Value); } } @@ -100,7 +100,7 @@ public void AddFilter(IEvidenceKeyFilter filter) /// public override bool Include(string key) { - // First check the white list as this will be faster than + // First check the inclusionList as this will be faster than // almost anything else (check against a hash table) bool include = base.Include(key); diff --git a/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterWhitelist.cs b/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterWhitelist.cs index 71d024e7..db403914 100644 --- a/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterWhitelist.cs +++ b/FiftyOne.Pipeline.Core/Data/EvidenceKeyFilterWhitelist.cs @@ -29,7 +29,7 @@ namespace FiftyOne.Pipeline.Core.Data { /// - /// This evidence filter will only include keys that are on a whitelist + /// This evidence filter will only include keys that are in a list /// that is specified at construction time. /// public class EvidenceKeyFilterWhitelist : IEvidenceKeyFilter @@ -38,13 +38,13 @@ public class EvidenceKeyFilterWhitelist : IEvidenceKeyFilter // Extending classes can make direct use of these fields. /// - /// The dictionary containing all keys in the whitelist and the - /// order of precedence. + /// The dictionary containing all keys to be included by the filter + /// and the order of precedence. /// - protected Dictionary _whitelist; + protected Dictionary _inclusionList; /// /// The equality comparer that is used to determine if a supplied - /// string key is in the whitelist or not. + /// string key is in the inclusion list or not. /// By default, a case insensitive comparison is used. /// protected IEqualityComparer _comparer = @@ -52,19 +52,19 @@ public class EvidenceKeyFilterWhitelist : IEvidenceKeyFilter #pragma warning restore CA1051 // Do not declare visible instance fields /// - /// Get the keys in the white list as a read only dictionary. + /// Get the keys in the inclusion list as a read only dictionary. /// public IReadOnlyDictionary Whitelist { get { - return new ReadOnlyDictionary(_whitelist); + return new ReadOnlyDictionary(_inclusionList); } } /// /// Get the equality comparer that is used to determine if a supplied - /// string key is in the whitelist or not. + /// string key is in the inclusion list or not. /// public IEqualityComparer Comparer => _comparer; @@ -73,19 +73,19 @@ public IReadOnlyDictionary Whitelist /// The filter will be case-insensitive. For a case-sensitive filter /// use the overload that takes an . /// - /// + /// /// The list of evidence keys that is filter will include. /// By default, all keys will have the same order of precedence. /// - public EvidenceKeyFilterWhitelist(List whitelist) + public EvidenceKeyFilterWhitelist(List inclusionList) { - PopulateFromList(whitelist); + PopulateFromList(inclusionList); } /// /// Constructor /// - /// + /// /// The list of evidence keys that is filter will include. /// By default, all keys will have the same order of precedence. /// @@ -93,11 +93,11 @@ public EvidenceKeyFilterWhitelist(List whitelist) /// Comparator to use when comparing the keys. /// public EvidenceKeyFilterWhitelist( - List whitelist, + List inclusionList, IEqualityComparer comparer) { _comparer = comparer; - PopulateFromList(whitelist); + PopulateFromList(inclusionList); } /// @@ -105,20 +105,20 @@ public EvidenceKeyFilterWhitelist( /// The filter will be case-insensitive. For a case-sensitive filter /// use the overload that takes an . /// - /// + /// /// The dictionary of evidence keys that is filter will include. /// The order of precedence of each key is given by the value of /// the key/value pair. /// - public EvidenceKeyFilterWhitelist(Dictionary whitelist) + public EvidenceKeyFilterWhitelist(Dictionary inclusionList) { - PopulateFromDictionary(whitelist); + PopulateFromDictionary(inclusionList); } /// /// Constructor /// - /// + /// /// The dictionary of evidence keys that is filter will include. /// The order of precedence of each key is given by the value of /// the key/value pair. @@ -127,22 +127,22 @@ public EvidenceKeyFilterWhitelist(Dictionary whitelist) /// Comparator to use when comparing the keys. /// public EvidenceKeyFilterWhitelist( - Dictionary whitelist, + Dictionary inclusionList, IEqualityComparer comparer) { _comparer = comparer; - PopulateFromDictionary(whitelist); + PopulateFromDictionary(inclusionList); } - private void PopulateFromList(List whitelist) + private void PopulateFromList(List inclusionList) { - _whitelist = whitelist.ToDictionary(w => w, w => 0, _comparer); + _inclusionList = inclusionList.ToDictionary(w => w, w => 0, _comparer); } - private void PopulateFromDictionary(Dictionary whitelist) + private void PopulateFromDictionary(Dictionary inclusionList) { - _whitelist = whitelist.ToDictionary(w => w.Key, w => w.Value, _comparer); + _inclusionList = inclusionList.ToDictionary(w => w.Key, w => w.Value, _comparer); } /// @@ -156,7 +156,7 @@ private void PopulateFromDictionary(Dictionary whitelist) /// public virtual bool Include(string key) { - return _whitelist.ContainsKey(key); + return _inclusionList.ContainsKey(key); } /// @@ -168,13 +168,13 @@ public virtual bool Include(string key) /// /// The order, where lower values indicate a higher order of /// precedence. - /// Null if the key is not in the white list. + /// Null if the key is not in the inclusion list. /// public virtual int? Order(string key) { int? result = 0; int temp; - if(_whitelist.TryGetValue(key, out temp) == false) + if(_inclusionList.TryGetValue(key, out temp) == false) { result = null; } diff --git a/FiftyOne.Pipeline.Core/Data/IEvidenceKeyFilter.cs b/FiftyOne.Pipeline.Core/Data/IEvidenceKeyFilter.cs index fc21706d..21537df7 100644 --- a/FiftyOne.Pipeline.Core/Data/IEvidenceKeyFilter.cs +++ b/FiftyOne.Pipeline.Core/Data/IEvidenceKeyFilter.cs @@ -54,7 +54,7 @@ public interface IEvidenceKeyFilter /// /// The order, where lower values indicate a higher order of /// precedence. - /// Null if the key is not in the white list. + /// Null if the key is not recognized. /// int? Order(string key); } diff --git a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs index 50bab81b..ec6f08b4 100644 --- a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs +++ b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs @@ -39,6 +39,7 @@ using FiftyOne.Pipeline.Engines.FiftyOne; using System.Collections.Concurrent; using Newtonsoft.Json.Serialization; +using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; namespace FiftyOne.Pipeline.JsonBuilder.FlowElement { @@ -55,6 +56,13 @@ public class JsonBuilderElement : FlowElementBase, IJsonBuilderElement { + /// + /// The element data key used by default for this element. + /// +#pragma warning disable CA1707 // Identifiers should not contain underscores + public const string DEFAULT_ELEMENT_DATA_KEY = "json-builder"; +#pragma warning restore CA1707 // Identifiers should not contain underscores + /// /// This contract resolver ensurers that property names /// are always converted to lowercase. @@ -73,8 +81,8 @@ protected override string ResolvePropertyName(string propertyName) private EvidenceKeyFilterWhitelist _evidenceKeyFilter; private List _properties; - private List _blacklist; - private HashSet _elementBlacklist; + private List _propertyExclusionlist; + private HashSet _elementExclusionList; /// /// Contains configuration information relating to a particular @@ -156,12 +164,16 @@ public JsonBuilderElement( this, "json", typeof(string), true) }; - // Blacklist of properties which should not be added to the Json. - _blacklist = new List() { "products", "properties" }; - // Blacklist of the element data keys of elements that should + // List of properties which should not be added to the Json. + _propertyExclusionlist = new List() { "products", "properties" }; + // List of the element data keys of elements that should // not be added to the Json. - _elementBlacklist = new HashSet(StringComparer.OrdinalIgnoreCase) - { "cloud-response", "json-builder" }; + _elementExclusionList = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "cloud-response", + DEFAULT_ELEMENT_DATA_KEY, + SetHeadersElement.DEFAULT_ELEMENT_DATA_KEY + }; _pipelineConfigs = new ConcurrentDictionary(); @@ -172,7 +184,7 @@ public JsonBuilderElement( /// The key to identify this engine's element data instance /// within . /// - public override string ElementDataKey => "json-builder"; + public override string ElementDataKey => DEFAULT_ELEMENT_DATA_KEY; /// /// A filter that identifies the evidence items that this @@ -363,7 +375,7 @@ protected virtual Dictionary GetAllProperties( Dictionary allProperties = new Dictionary(); foreach (var element in data.ElementDataAsDictionary().Where(elementData => - _elementBlacklist.Contains(elementData.Key) == false)) + _elementExclusionList.Contains(elementData.Key) == false)) { if (allProperties.ContainsKey(element.Key.ToLowerInvariant()) == false) { diff --git a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs index a143f088..9d8d6bfe 100644 --- a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs +++ b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs @@ -120,16 +120,16 @@ public void JsonBuilder_MaxIterations() /// /// Check that entries will not appear in the output - /// for blacklisted elements. + /// for elements in the exclusion list. /// [TestMethod] - public void JsonBuilder_ElementBlacklist() + public void JsonBuilder_ElementExclusionlist() { var json = TestIteration(1, new Dictionary() { { "test", _elementDataMock.Object }, { "cloud-response", _elementDataMock.Object }, - { "json-builder", _elementDataMock.Object } + { JsonBuilderElement.DEFAULT_ELEMENT_DATA_KEY, _elementDataMock.Object } }); Assert.IsTrue(IsExpectedJson(json)); @@ -386,7 +386,7 @@ public class JsonData [JsonProperty("empty-aspect")] public EmptyAspect EmptyAspect { get; set; } - [JsonProperty("json-builder")] + [JsonProperty(JsonBuilderElement.DEFAULT_ELEMENT_DATA_KEY)] public JsonBuilder JsonBuilder { get; set; } } diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/Data/EvidenceKeyFilterShareUsage.cs b/FiftyOne.Pipeline.Engines.FiftyOne/Data/EvidenceKeyFilterShareUsage.cs index 9c041725..58ba8295 100644 --- a/FiftyOne.Pipeline.Engines.FiftyOne/Data/EvidenceKeyFilterShareUsage.cs +++ b/FiftyOne.Pipeline.Engines.FiftyOne/Data/EvidenceKeyFilterShareUsage.cs @@ -41,7 +41,7 @@ namespace FiftyOne.Pipeline.Engines.FiftyOne.Data /// /// As this filter is generally inclusive, it will often cause far more /// evidence to be passed into a pipeline than the engine-specific - /// filters, which tend to be based on a white list such as + /// filters, which tend to be based on a list of values to include such as /// . /// public class EvidenceKeyFilterShareUsage : IEvidenceKeyFilter diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs index 50c3c30b..d6b159b0 100644 --- a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs +++ b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs @@ -43,6 +43,13 @@ public class SetHeadersElement : FlowElementBase, ISetHeadersElement { + /// + /// The element data key used by default for this element. + /// +#pragma warning disable CA1707 // Identifiers should not contain underscores + public const string DEFAULT_ELEMENT_DATA_KEY = "set-headers"; +#pragma warning restore CA1707 // Identifiers should not contain underscores + /// /// Contains configuration information relating to a particular /// pipeline. @@ -98,7 +105,7 @@ public SetHeadersElement( } /// - public override string ElementDataKey => "set-headers"; + public override string ElementDataKey => DEFAULT_ELEMENT_DATA_KEY; /// public override IEvidenceKeyFilter EvidenceKeyFilter => _evidenceKeyFilter; diff --git a/FiftyOne.Pipeline.Engines.TestHelpers/EmptyEngine.cs b/FiftyOne.Pipeline.Engines.TestHelpers/EmptyEngine.cs index 5f1651cf..39084c60 100644 --- a/FiftyOne.Pipeline.Engines.TestHelpers/EmptyEngine.cs +++ b/FiftyOne.Pipeline.Engines.TestHelpers/EmptyEngine.cs @@ -69,12 +69,12 @@ public void SetException(Exception exception) public override string ElementDataKey => "empty-aspect"; - private EvidenceKeyFilterWhitelist _evidnceWhitelist = + private EvidenceKeyFilterWhitelist _evidenceInclusionList = new EvidenceKeyFilterWhitelist(new List() { "test.value" }); - public override IEvidenceKeyFilter EvidenceKeyFilter => _evidnceWhitelist; + public override IEvidenceKeyFilter EvidenceKeyFilter => _evidenceInclusionList; public override string DataSourceTier => throw new NotImplementedException(); diff --git a/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs b/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs index 5646d11e..4aae1265 100644 --- a/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs +++ b/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs @@ -161,7 +161,7 @@ public enum AutoUpdateStatus /// /// AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS, /// - /// 51Degrees server responded with 403 meaning key is blacklisted. + /// 51Degrees server responded with 403, meaning key is revoked. /// AUTO_UPDATE_ERR_403_FORBIDDEN, /// diff --git a/FiftyOne.Pipeline.Web.Shared/Services/ClientsidePropertyService.cs b/FiftyOne.Pipeline.Web.Shared/Services/ClientsidePropertyService.cs index c50e8f96..197e3a45 100644 --- a/FiftyOne.Pipeline.Web.Shared/Services/ClientsidePropertyService.cs +++ b/FiftyOne.Pipeline.Web.Shared/Services/ClientsidePropertyService.cs @@ -98,10 +98,10 @@ public ClientsidePropertyService( // get all HTTP header evidence keys from white list // and add them to the headers that could affect the // generated JavaScript. - var whitelist = filter as EvidenceKeyFilterWhitelist; - if (whitelist != null) + var inclusionList = filter as EvidenceKeyFilterWhitelist; + if (inclusionList != null) { - headersAffectingJavaScript.AddRange(whitelist.Whitelist + headersAffectingJavaScript.AddRange(inclusionList.Whitelist .Where(entry => entry.Key.StartsWith( Core.Constants.EVIDENCE_HTTPHEADER_PREFIX + Core.Constants.EVIDENCE_SEPERATOR, StringComparison.OrdinalIgnoreCase)) From 2eceeae73398303e880a0203aff3229090607b5a Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Wed, 28 Apr 2021 14:20:01 +0000 Subject: [PATCH 07/14] CLEANUP: Use constant for 'SetHeader' and associated string lengths. CLEANUP: Use constant for 'SetHeader' and associated string lengths. --- .../FlowElements/SetHeadersElement.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs index d6b159b0..0110eecb 100644 --- a/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs +++ b/FiftyOne.Pipeline.Engines.FiftyOne/FlowElements/SetHeadersElement.cs @@ -48,6 +48,8 @@ public class SetHeadersElement : /// #pragma warning disable CA1707 // Identifiers should not contain underscores public const string DEFAULT_ELEMENT_DATA_KEY = "set-headers"; + + private const string SET_HEADER_PROPERTY_PREFIX = "SetHeader"; #pragma warning restore CA1707 // Identifiers should not contain underscores /// @@ -208,7 +210,7 @@ private PipelineConfig PopulateConfig(IPipeline pipeline) foreach (var element in pipeline.ElementAvailableProperties) { foreach (var property in element.Value.Where(p => - p.Key.StartsWith("SetHeader", StringComparison.OrdinalIgnoreCase))) + p.Key.StartsWith(SET_HEADER_PROPERTY_PREFIX, StringComparison.OrdinalIgnoreCase))) { PropertyDetails details = new PropertyDetails() { @@ -224,7 +226,7 @@ private PipelineConfig PopulateConfig(IPipeline pipeline) private static string GetResponseHeaderName(string propertyName) { - if(propertyName.StartsWith("SetHeader", + if(propertyName.StartsWith(SET_HEADER_PROPERTY_PREFIX, StringComparison.Ordinal) == false) { throw new ArgumentException(string.Format( @@ -233,7 +235,7 @@ private static string GetResponseHeaderName(string propertyName) propertyName), nameof(propertyName)); } - if(propertyName.Length < 11) + if(propertyName.Length < SET_HEADER_PROPERTY_PREFIX.Length + 2) { throw new ArgumentException(string.Format( CultureInfo.InvariantCulture, @@ -243,7 +245,7 @@ private static string GetResponseHeaderName(string propertyName) } int nextUpper = -1; - for(int i = 10; i < propertyName.Length; i++) + for(int i = SET_HEADER_PROPERTY_PREFIX.Length + 1; i < propertyName.Length; i++) { if (char.IsUpper(propertyName[i])) { From cc4693edd52f342b6be853be4324ccbc202f0de1 Mon Sep 17 00:00:00 2001 From: Ben Shillito Date: Thu, 20 May 2021 15:15:26 +0000 Subject: [PATCH 08/14] FEAT/BUG: Various features and a bug fix BUG: view component was looking under the wrong namespace for PipelineWebIntegrationOption. FEAT: ASP.NET Framework Pipeline now adds data update service and missing property service by default. This means that data files can be automatically updated by supplying the correct config, as in ASP.NET Core. FEAT: Add the SetHeaders element automatically if it is not configured in the Framework web implementation. --- .../FlowElements/PipelineBuilder.cs | 91 +++++++++++++- .../Services/FiftyOneServiceProvider.cs | 57 +++++++++ .../Services/IDataUpdateService.cs | 1 + .../Services/IMissingPropertyService.cs | 1 + .../FlowElements/PipelineBuilderTests.cs | 116 +++++++++++++++++- .../HelperClasses/RequiredServiceElement.cs | 47 +++++++ .../RequiredServiceElementBuilder.cs | 53 ++++++++ .../WebPipeline.cs | 34 ++++- .../Components/FiftyOneJS/Default.cshtml | 2 +- 9 files changed, 393 insertions(+), 9 deletions(-) create mode 100644 FiftyOne.Pipeline.Core/Services/FiftyOneServiceProvider.cs create mode 100644 Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElement.cs create mode 100644 Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElementBuilder.cs diff --git a/FiftyOne.Pipeline.Core/FlowElements/PipelineBuilder.cs b/FiftyOne.Pipeline.Core/FlowElements/PipelineBuilder.cs index f1e9f13c..0ddd695a 100644 --- a/FiftyOne.Pipeline.Core/FlowElements/PipelineBuilder.cs +++ b/FiftyOne.Pipeline.Core/FlowElements/PipelineBuilder.cs @@ -23,6 +23,7 @@ using FiftyOne.Pipeline.Core.Attributes; using FiftyOne.Pipeline.Core.Configuration; using FiftyOne.Pipeline.Core.Exceptions; +using FiftyOne.Pipeline.Core.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; @@ -246,7 +247,8 @@ private void AddElementToList( } object builderInstance = null; - if (_services != null) + if (_services != null && + _services.GetType().Equals(typeof(FiftyOneServiceProvider)) == false) { // Try to get a a builder instance from the service collection. builderInstance = _services.GetRequiredService(builderType); @@ -434,6 +436,75 @@ private void AddParallelElementsToList( elements.Add(parallelInstance); } + /// + /// Get the services required for the constructor, and call it with them. + /// + /// + /// The constructor to call. + /// + /// + /// Instance returned by the constructor. + /// + private object CallConstructorWithServicesForAssemblies( + ConstructorInfo constructor) + { + ParameterInfo[] parameters = constructor.GetParameters(); + object[] services = new object[parameters.Length]; + for (int i = 0; i < parameters.Length; i++) + { + if (parameters[i].ParameterType.Equals(typeof(ILoggerFactory))) + { + services[i] = LoggerFactory; + } + else + { + services[i] = _services.GetService(parameters[i].ParameterType); + } + } + return Activator.CreateInstance(constructor.DeclaringType, services); + } + + + /// + /// Get the best constructor for the list of constructors. Best meaning + /// the constructor with the most parameters which can be fulfilled. + /// + /// + /// Constructors to get the best of. + /// + /// + /// Best constructor or null if none have parameters that can be + /// fulfilled. + /// + private ConstructorInfo GetBestConstructorForAssemblies( + IEnumerable constructors) + { + ConstructorInfo bestConstructor = null; + foreach (var constructor in constructors) + { + if (bestConstructor == null || + constructor.GetParameters().Length > + bestConstructor.GetParameters().Length) + { + var hasServices = true; + foreach (var param in constructor.GetParameters()) + { + if (param.ParameterType.Equals(typeof(ILoggerFactory)) == false && + _services.GetService(param.ParameterType) == null) + { + hasServices = false; + break; + } + } + if (hasServices == true) + { + bestConstructor = constructor; + } + } + } + return bestConstructor; + } + /// /// Instantiate a new builder instance from the assemblies which are /// currently loaded. @@ -450,9 +521,15 @@ private object GetBuilderFromAssemlies(Type builderType) var loggerConstructors = builderType.GetConstructors() .Where(c => c.GetParameters().Length == 1 && c.GetParameters()[0].ParameterType == typeof(ILoggerFactory)); - + var serviceConstructors = builderType.GetConstructors() + .Where(c => c.GetParameters().Length > 1 && + c.GetParameters().All(p => p.ParameterType.Equals(typeof(ILoggerFactory)) || + (_services != null && + _services.GetService(p.ParameterType) != null))); + if (defaultConstructors.Any() == false && - loggerConstructors.Any() == false) + loggerConstructors.Any() == false && + serviceConstructors.Any() == false) { return null; } @@ -460,7 +537,13 @@ private object GetBuilderFromAssemlies(Type builderType) // Create the builder instance using the constructor with a logger // factory, or the default constructor if one taking a logger // factory is not available. - if (loggerConstructors.Any()) + if (serviceConstructors.Any() && + GetBestConstructorForAssemblies(serviceConstructors) != null) + { + return CallConstructorWithServicesForAssemblies( + GetBestConstructorForAssemblies(serviceConstructors)); + } + else if (loggerConstructors.Any()) { return Activator.CreateInstance(builderType, LoggerFactory); } diff --git a/FiftyOne.Pipeline.Core/Services/FiftyOneServiceProvider.cs b/FiftyOne.Pipeline.Core/Services/FiftyOneServiceProvider.cs new file mode 100644 index 00000000..25c9b486 --- /dev/null +++ b/FiftyOne.Pipeline.Core/Services/FiftyOneServiceProvider.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Core.Services +{ + /// + /// Basic implementation of . + /// An instance contains a list of services which can be added + /// to. + /// + public class FiftyOneServiceProvider : IServiceProvider + { + private readonly IList _services = new List(); + + /// + /// Add a service instance to the provider. This builds the + /// collection used to return services from the GetService + /// method. + /// + /// + /// Service instance to add. + /// + public void AddService(object service) + { + _services.Add(service); + } + + + /// + /// Get the service from the service collection if it exists, otherwise + /// return null. + /// Note that if more than one instance implementing the same service + /// is added to the services, the first will be returned. + /// + /// + /// The service type to be returned. + /// + /// + /// Service or null. + /// + public object GetService(Type serviceType) + { + if (serviceType != null) + { + foreach (var service in _services) + { + if (serviceType.IsAssignableFrom(service.GetType())) + { + return service; + } + } + } + return null; + } + } +} diff --git a/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs b/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs index 4aae1265..a08c1d64 100644 --- a/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs +++ b/FiftyOne.Pipeline.Engines/Services/IDataUpdateService.cs @@ -20,6 +20,7 @@ * such notice(s) shall fulfill the requirements of that article. * ********************************************************************* */ +using FiftyOne.Pipeline.Core.Services; using FiftyOne.Pipeline.Engines.Configuration; using FiftyOne.Pipeline.Engines.Data; using FiftyOne.Pipeline.Engines.Exceptions; diff --git a/FiftyOne.Pipeline.Engines/Services/IMissingPropertyService.cs b/FiftyOne.Pipeline.Engines/Services/IMissingPropertyService.cs index 98aba507..5abfae5e 100644 --- a/FiftyOne.Pipeline.Engines/Services/IMissingPropertyService.cs +++ b/FiftyOne.Pipeline.Engines/Services/IMissingPropertyService.cs @@ -20,6 +20,7 @@ * such notice(s) shall fulfill the requirements of that article. * ********************************************************************* */ +using FiftyOne.Pipeline.Core.Services; using FiftyOne.Pipeline.Engines.FlowElements; using System; using System.Collections.Generic; diff --git a/Tests/FiftyOne.Pipeline.Core.Tests/FlowElements/PipelineBuilderTests.cs b/Tests/FiftyOne.Pipeline.Core.Tests/FlowElements/PipelineBuilderTests.cs index 63de069a..e4b1182f 100644 --- a/Tests/FiftyOne.Pipeline.Core.Tests/FlowElements/PipelineBuilderTests.cs +++ b/Tests/FiftyOne.Pipeline.Core.Tests/FlowElements/PipelineBuilderTests.cs @@ -23,12 +23,16 @@ using FiftyOne.Pipeline.Core.Configuration; using FiftyOne.Pipeline.Core.Exceptions; using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Core.Services; using FiftyOne.Pipeline.Core.Tests.HelperClasses; +using FiftyOne.Pipeline.Engines.Services; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; using System.Collections.Generic; - +using System.Net.Http; + namespace FiftyOne.Pipeline.Core.Tests.FlowElements { /// @@ -717,6 +721,116 @@ public void PipelineBuilder_BuildFromConfiguration_ListFromStringSingleEntry() VerifyListSplitterElementPipeline(opts, SplitOption.Pipe); } + /// + /// Test that if the services declared in a builders constructor are not + /// provided, but there are other constructors available, that the correct + /// constructor is called. + /// This is specifically testing the logic used when a ServiceCollection + /// is not available e.g. in ASP.NET Framework. + /// + [TestMethod] + public void PipelineBuilder_BuildFromConfiguration_AssemblyServices_NotAvailable() + { + var element = new ElementOptions() + { + BuilderName = "RequiredService" + }; + + // Create configuration object. + PipelineOptions opts = new PipelineOptions(); + opts.Elements = new List + { + element + }; + + // Pass the configuration to the builder to create the pipeline. + var pipeline = new PipelineBuilder(_loggerFactory, new FiftyOneServiceProvider()) + .BuildFromConfiguration(opts); + + Assert.IsNotNull(pipeline.GetElement().LoggerFactory); + Assert.IsNull(pipeline.GetElement().Service); + Assert.IsNull(pipeline.GetElement().UpdateService); + } + + /// + /// Test that if the services declared in a builders constructor are provided, + /// but there are other constructors available, that the correct constructor + /// is called. + /// This is specifically testing the logic used when a ServiceCollection + /// is not available e.g. in ASP.NET Framework. + /// + [TestMethod] + public void PipelineBuilder_BuildFromConfiguration_AssemblyServices_Available() + { + var element = new ElementOptions() + { + BuilderName = "RequiredService" + }; + + // Create configuration object. + PipelineOptions opts = new PipelineOptions(); + opts.Elements = new List + { + element + }; + + var service = new RequiredServiceElementBuilder.EmptyService(); + var services = new FiftyOneServiceProvider(); + services.AddService(service); + + // Pass the configuration to the builder to create the pipeline. + var pipeline = new PipelineBuilder(_loggerFactory, services) + .BuildFromConfiguration(opts); + + Assert.IsNotNull(pipeline.GetElement().LoggerFactory); + Assert.IsNotNull(pipeline.GetElement().Service); + Assert.AreEqual(service, pipeline.GetElement().Service); + Assert.IsNull(pipeline.GetElement().UpdateService); + } + + /// + /// Test that if the services declared in a builders constructor are provided, + /// but there are other constructors available, that the correct constructor + /// is called. This include the DataUpdateService to test the specific + /// scenario in addition to the general one. + /// This is specifically testing the logic used when a ServiceCollection + /// is not available e.g. in ASP.NET Framework. + /// + [TestMethod] + public void PipelineBuilder_BuildFromConfiguration_AssemblyServices_MultiAvailable() + { + var element = new ElementOptions() + { + BuilderName = "RequiredService" + }; + + // Create configuration object. + PipelineOptions opts = new PipelineOptions(); + opts.Elements = new List + { + element + }; + + var service = new RequiredServiceElementBuilder.EmptyService(); + var httpClient = new Mock(); + var updateService = new DataUpdateService( + new Mock>().Object, + httpClient.Object); + var services = new FiftyOneServiceProvider(); + services.AddService(service); + services.AddService(updateService); + + // Pass the configuration to the builder to create the pipeline. + var pipeline = new PipelineBuilder(_loggerFactory, services) + .BuildFromConfiguration(opts); + + Assert.IsNotNull(pipeline.GetElement().LoggerFactory); + Assert.IsNotNull(pipeline.GetElement().Service); + Assert.AreEqual(service, pipeline.GetElement().Service); + Assert.IsNotNull(pipeline.GetElement().UpdateService); + Assert.AreEqual(updateService, pipeline.GetElement().UpdateService); + } + private enum SplitOption { Comma, Pipe, CommaMaxLengthThree, CommaAndPipe, PipeAndQuote } private void VerifyListSplitterElementPipeline( diff --git a/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElement.cs b/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElement.cs new file mode 100644 index 00000000..d983667d --- /dev/null +++ b/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElement.cs @@ -0,0 +1,47 @@ +using FiftyOne.Pipeline.Core.Data; +using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Engines.Services; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Text; +using static FiftyOne.Pipeline.Core.Tests.HelperClasses.RequiredServiceElementBuilder; + +namespace FiftyOne.Pipeline.Core.Tests.HelperClasses +{ + public class RequiredServiceElement : FlowElementBase + { + + public ILoggerFactory LoggerFactory { get; private set; } = null; + public EmptyService Service { get; private set; } = null; + public IDataUpdateService UpdateService { get; private set; } = null; + + public override string ElementDataKey => "requiredservice"; + + public override IEvidenceKeyFilter EvidenceKeyFilter => new EvidenceKeyFilterWhitelist(new List()); + + public override IList Properties => new List(); + + public RequiredServiceElement( + ILoggerFactory loggerFactory, + EmptyService service, + IDataUpdateService updateService) + : base(loggerFactory.CreateLogger()) + { + LoggerFactory = loggerFactory; + Service = service; + UpdateService = updateService; + } + protected override void ProcessInternal(IFlowData data) + { + } + + protected override void ManagedResourcesCleanup() + { + } + + protected override void UnmanagedResourcesCleanup() + { + } + } +} diff --git a/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElementBuilder.cs b/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElementBuilder.cs new file mode 100644 index 00000000..beeb15f0 --- /dev/null +++ b/Tests/FiftyOne.Pipeline.Core.Tests/HelperClasses/RequiredServiceElementBuilder.cs @@ -0,0 +1,53 @@ +using FiftyOne.Pipeline.Core.Attributes; +using FiftyOne.Pipeline.Core.Services; +using FiftyOne.Pipeline.Engines.Services; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Core.Tests.HelperClasses +{ + [AlternateName("RequiredService")] + public class RequiredServiceElementBuilder + { + public class EmptyService + { + + } + + public ILoggerFactory LoggerFactory { get; private set; } = null; + public EmptyService Service { get; private set; } = null; + public IDataUpdateService UpdateService { get; private set; } = null; + + public RequiredServiceElementBuilder( + ILoggerFactory loggerFactory, + EmptyService service) + { + LoggerFactory = loggerFactory; + Service = service; + } + public RequiredServiceElementBuilder( + ILoggerFactory loggerFactory, + EmptyService service, + IDataUpdateService updateService) + { + LoggerFactory = loggerFactory; + Service = service; + UpdateService = updateService; + } + public RequiredServiceElementBuilder(ILoggerFactory loggerFactory) + { + LoggerFactory = loggerFactory; + } + public RequiredServiceElementBuilder() + { + + } + + public RequiredServiceElement Build() + { + return new RequiredServiceElement(LoggerFactory, Service, UpdateService); + } + } +} diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs index 35031cd1..f19ec1fe 100644 --- a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs +++ b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs @@ -24,7 +24,9 @@ using FiftyOne.Pipeline.Core.Data; using FiftyOne.Pipeline.Core.Exceptions; using FiftyOne.Pipeline.Core.FlowElements; +using FiftyOne.Pipeline.Core.Services; using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using FiftyOne.Pipeline.Engines.Services; using FiftyOne.Pipeline.JavaScriptBuilder.FlowElement; using FiftyOne.Pipeline.JsonBuilder.FlowElement; using FiftyOne.Pipeline.Web.Framework.Configuration; @@ -110,7 +112,7 @@ private WebPipeline() .Build(); _options = new PipelineWebIntegrationOptions(); config.Bind("PipelineOptions", _options); - + if (_options == null || _options.Elements == null) { @@ -187,9 +189,35 @@ private WebPipeline() javascriptConfig.Single().BuildParameters.Add("EndPoint", "/51dpipeline/json"); } } - } + } + + // Add the set headers + var setHeadersConfig = _options.Elements.Where(e => + e.BuilderName.IndexOf(nameof(SetHeadersElement), + StringComparison.OrdinalIgnoreCase) >= 0); + if (setHeadersConfig.Any() == false) + { + // The set headers element is not included, so add it. + // Make sure it's added as the last element. + _options.Elements.Add(new ElementOptions() + { + BuilderName = nameof(SetHeadersElement) + }); + } + + // Set up common services. + var loggerFactory = new LoggerFactory(); + var updateService = new DataUpdateService( + loggerFactory.CreateLogger(), + new System.Net.Http.HttpClient()); + var services = new FiftyOneServiceProvider(); + // Add data update and missing property services. + services.AddService(updateService); + services.AddService(MissingPropertyService.Instance); - Pipeline = new PipelineBuilder(new LoggerFactory()) + Pipeline = new PipelineBuilder( + loggerFactory, + services) .BuildFromConfiguration(_options); } diff --git a/Web Integration/FiftyOne.Pipeline.Web/Views/Shared/Components/FiftyOneJS/Default.cshtml b/Web Integration/FiftyOne.Pipeline.Web/Views/Shared/Components/FiftyOneJS/Default.cshtml index 660f08d0..e730ac99 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/Views/Shared/Components/FiftyOneJS/Default.cshtml +++ b/Web Integration/FiftyOne.Pipeline.Web/Views/Shared/Components/FiftyOneJS/Default.cshtml @@ -1,4 +1,4 @@ -@model Microsoft.Extensions.Options.IOptions +@model Microsoft.Extensions.Options.IOptions @if (Model.Value.ClientSideEvidenceEnabled) { From 6f3c4977ce46654f102b5202d3c447b5f58be465 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Mon, 24 May 2021 08:57:37 +0000 Subject: [PATCH 09/14] BUG: Addressed some issues raised by users and internal testing of 4.3.0 beta. - BUG: Should only try and read from the 'Form' property on an HTTP request after checking if the content type is correct. This addresses https://github.com/51Degrees/pipeline-dotnet/issues/5. - BUG: ASP.NET Framework integration was not reading form parameters. This has now been corrected. - TEST: Add some tests to verify that the functionality to add form parameters to evidence in the WebRequestEvidenceService is functioning correctly. - TEST: Add a new JsonBuilderElement test to verify the javascriptProperties element in the JSON for properties with type JavaScript, AspectPropertyValue or IAspectPropertyValue. - TEST: Modify the JsonBuilderElement tests to verify that the delayed evidence functionality works correctly is the property type is JavaScript, AspectPropertyValue or IAspectPropertyValue. - BUG: Correct a bug that stopped properties from being included in the javascriptProperties list if they had type AspectPropertyValue. - BUG: JsonBuilderElement was not correctly checking JavaScript types when getting delayed execution properties. This has now been corrected. Also moved the type check to a static utility class. Related work items: #4427, #4450 --- .../FlowElement/JsonBuilderElement.cs | 8 +- .../JsonBuilderElementTests.cs | 134 +++++++++++----- FiftyOne.Pipeline.Engines/Utils.cs | 42 +++++ FiftyOne.Pipeline.Web.Shared/Constants.cs | 32 ++++ .../WebPipeline.cs | 12 ++ .../Services/WebRequestEvidenceService.cs | 4 +- .../WebRequestEvidenceServiceTest.cs | 143 +++++++++++++----- 7 files changed, 294 insertions(+), 81 deletions(-) create mode 100644 FiftyOne.Pipeline.Engines/Utils.cs create mode 100644 FiftyOne.Pipeline.Web.Shared/Constants.cs diff --git a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs index ec6f08b4..09fdb5f0 100644 --- a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs +++ b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElement/FlowElement/JsonBuilderElement.cs @@ -40,6 +40,7 @@ using System.Collections.Concurrent; using Newtonsoft.Json.Serialization; using FiftyOne.Pipeline.Engines.FiftyOne.FlowElements; +using FiftyOne.Pipeline.Engines; namespace FiftyOne.Pipeline.JsonBuilder.FlowElement { @@ -635,9 +636,8 @@ protected virtual IList GetJavaScriptProperties( var javascriptPropertiesEnumerable = data.GetWhere( p => p.Type != null && - (p.Type.Equals(typeof(JavaScript)) || - p.Type.Equals(typeof(IAspectPropertyValue)))) - .Where(p => availableProperties.Contains(p.Key)); + Utils.IsTypeOrAspectPropertyValue(p.Type)) + .Where(p => availableProperties.Contains(p.Key)); List javascriptPropeties = new List(); foreach (var property in javascriptPropertiesEnumerable) @@ -702,7 +702,7 @@ private IEnumerable GetDelayedPropertyNames ( // Return the names of any delayed execution properties. foreach(var property in properties.Where(p => p.DelayExecution && - p.Type == typeof(JavaScript))) + Utils.IsTypeOrAspectPropertyValue(p.Type))) { yield return $"{dataPath}{Core.Constants.EVIDENCE_SEPERATOR}" + $"{property.Name.ToLowerInvariant()}"; diff --git a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs index 9d8d6bfe..126cd27a 100644 --- a/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs +++ b/FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JsonBuilderElementTests/JsonBuilderElementTests.cs @@ -48,12 +48,20 @@ namespace FiftyOne.Pipeline.JsonBuilderElementTests [TestClass] public class JsonBuilderElementTests { + public enum JsPropertyType + { + JavaScript, + IAspectPropertyValue, + AspectPropertyValue + } + private IJsonBuilderElement _jsonBuilderElement; private Mock _elementDataMock; private ILoggerFactory _loggerFactory; private Mock _pipeline; private Mock _testEngine; + private Dictionary> _propertyMetaData; [TestInitialize] public void Init() @@ -75,8 +83,8 @@ public void Init() _pipeline = new Mock(); _pipeline.Setup(p => p.GetHashCode()).Returns(1); - var propertyMetaData = new Dictionary>(); - _pipeline.Setup(p => p.ElementAvailableProperties).Returns(propertyMetaData); + _propertyMetaData = new Dictionary>(); + _pipeline.Setup(p => p.ElementAvailableProperties).Returns(_propertyMetaData); } /// @@ -89,7 +97,33 @@ public void JsonBuilder_ValidJson() Assert.IsTrue(IsExpectedJson(json)); } - + + /// + /// Check that the JSON element removes JavaScript properties from the + /// response after max number of iterations has been reached. + /// + [DataTestMethod] + [DataRow(JsPropertyType.JavaScript)] + [DataRow(JsPropertyType.AspectPropertyValue)] + [DataRow(JsPropertyType.IAspectPropertyValue)] + public void JsonBuilder_JsProperty(JsPropertyType propertyType) + { + _elementDataMock.Setup(ed => ed.AsDictionary()). + Returns(new Dictionary() { + { "jsproperty", "var = 'some js code';" } + }); + + var testElementMetaData = new Dictionary(); + var p1 = new ElementPropertyMetaData(_testEngine.Object, "jsproperty", GetTypeFromEnum(propertyType), true, ""); + testElementMetaData.Add("jsproperty", p1); + _propertyMetaData.Add("test", testElementMetaData); + + var json = TestIteration(1); + Assert.IsTrue(ContainsJavaScriptProperties(json), + "The 'javascriptProperties' element is missing from the JSON. " + + "Complete JSON: " + Environment.NewLine + json); + } + /// /// Check that the JSON element removes JavaScript properties from the /// response after max number of iterations has been reached. @@ -97,23 +131,29 @@ public void JsonBuilder_ValidJson() [TestMethod] public void JsonBuilder_MaxIterations() { + _elementDataMock.Setup(ed => ed.AsDictionary()). + Returns(new Dictionary() { + { "jsproperty", "var = 'some js code';" } + }); + + var testElementMetaData = new Dictionary(); + var p1 = new ElementPropertyMetaData(_testEngine.Object, "jsproperty", typeof(AspectPropertyValue), true, ""); + testElementMetaData.Add("jsproperty", p1); + _propertyMetaData.Add("test", testElementMetaData); + for (var i = 0; true; i++) { - var jsProperties = new Dictionary() - { - { "test.jsproperty", new AspectPropertyValue(new JavaScript("var = 'some js code';")) } - }; - var json = TestIteration(i, null, jsProperties); + var json = TestIteration(i); var result = ContainsJavaScriptProperties(json); if (i >= Constants.MAX_JAVASCRIPT_ITERATIONS) { - Assert.IsFalse(result); + Assert.IsFalse(result, $"Failed on iteration {i}"); break; } else { - Assert.IsTrue(result); + Assert.IsTrue(result, $"Failed on iteration {i}"); } } } @@ -177,7 +217,6 @@ public void JsonBuilder_NestedProperties() // Configure the property meta-data as needed for // this test. - var propertyMetaData = new Dictionary>(); var testElementMetaData = new Dictionary(); var nestedMetaData = new List() { new ElementPropertyMetaData(_testEngine.Object, "value1", typeof(string), true), @@ -185,8 +224,7 @@ public void JsonBuilder_NestedProperties() }; var p1 = new ElementPropertyMetaData(_testEngine.Object, "property", typeof(List), true, "", nestedMetaData); testElementMetaData.Add("property", p1); - propertyMetaData.Add("test", testElementMetaData); - _pipeline.Setup(p => p.ElementAvailableProperties).Returns(propertyMetaData); + _propertyMetaData.Add("test", testElementMetaData); var json = TestIteration(1); @@ -222,18 +260,30 @@ public void JsonBuilder_NestedProperties() $"Complete JSON: " + Environment.NewLine + json); } + public static IEnumerable GetDelayedExecutionTestParameters + { + get + { + foreach (var entry in Enum.GetValues(typeof(JsPropertyType))) + { + yield return new object[] { true, true, entry }; + yield return new object[] { false, true, entry }; + yield return new object[] { true, false, entry }; + yield return new object[] { false, false, entry }; + } + } + } + /// /// Check that delayed execution and evidence properties values /// are populated correctly. /// [DataTestMethod] - [DataRow(true, true)] - [DataRow(false, true)] - [DataRow(true, false)] - [DataRow(false, false)] + [DynamicData(nameof(GetDelayedExecutionTestParameters))] public void JsonBuilder_DelayedExecution( bool delayExecution, - bool propertyValueNull) + bool propertyValueNull, + JsPropertyType jsPropertyType) { // If the flag is set then initialise the 'property' value to null. if (propertyValueNull) @@ -244,17 +294,15 @@ public void JsonBuilder_DelayedExecution( { "jsproperty", "var = 'some js code';" } }); } - + // Configure the property meta-data as needed for // this test. - var propertyMetaData = new Dictionary>(); var testElementMetaData = new Dictionary(); var p1 = new ElementPropertyMetaData(_testEngine.Object, "property", typeof(string), true, "", null, false, new List() { "jsproperty" }); testElementMetaData.Add("property", p1); - var p2 = new ElementPropertyMetaData(_testEngine.Object, "jsproperty", typeof(JavaScript), true, "", null, delayExecution); + var p2 = new ElementPropertyMetaData(_testEngine.Object, "jsproperty", GetTypeFromEnum(jsPropertyType), true, "", null, delayExecution); testElementMetaData.Add("jsproperty", p2); - propertyMetaData.Add("test", testElementMetaData); - _pipeline.Setup(p => p.ElementAvailableProperties).Returns(propertyMetaData); + _propertyMetaData.Add("test", testElementMetaData); // Run the test var json = TestIteration(1); @@ -297,7 +345,6 @@ public void JsonBuilder_MultipleEvidenceProperties() // jsproperty and jsproperty2. // jsproperty has delayed execution true and // jsproperty2 does not. - var propertyMetaData = new Dictionary>(); var testElementMetaData = new Dictionary(); var p1 = new ElementPropertyMetaData(_testEngine.Object, "property", typeof(string), true, "", null, false, new List() { "jsproperty", "jsproperty2" }); testElementMetaData.Add("property", p1); @@ -305,8 +352,7 @@ public void JsonBuilder_MultipleEvidenceProperties() testElementMetaData.Add("jsproperty", p2); var p3 = new ElementPropertyMetaData(_testEngine.Object, "jsproperty2", typeof(JavaScript), true, "", null, false); testElementMetaData.Add("jsproperty2", p2); - propertyMetaData.Add("test", testElementMetaData); - _pipeline.Setup(p => p.ElementAvailableProperties).Returns(propertyMetaData); + _propertyMetaData.Add("test", testElementMetaData); _elementDataMock.Setup(ed => ed.AsDictionary()). Returns(new Dictionary() { @@ -369,7 +415,6 @@ public void JsonBuilder_LazyLoading() flowData.Process(); Trace.WriteLine("Process complete"); - var jsonResult = flowData.Get(); Assert.IsNotNull(jsonResult); Assert.IsNotNull(jsonResult.Json); @@ -404,17 +449,12 @@ public class JsonBuilder } private string TestIteration(int iteration, - Dictionary data = null, - Dictionary jsProperties = null) + Dictionary data = null) { if(data == null) { data = new Dictionary() { { "test", _elementDataMock.Object } }; } - if(jsProperties == null) - { - jsProperties = new Dictionary(); - } var flowData = new Mock(); var _missingPropertyService = new Mock(); @@ -423,7 +463,13 @@ private string TestIteration(int iteration, string session = "somesessionid"; flowData.Setup(d => d.TryGetEvidence("query.session-id", out session)).Returns(true); flowData.Setup(d => d.TryGetEvidence("query.sequence", out iteration)).Returns(true); - flowData.Setup(d => d.GetWhere(It.IsAny>())).Returns(jsProperties); + flowData.Setup(d => d.GetWhere(It.IsAny>())).Returns( + (Func filter) => { + return _propertyMetaData + .SelectMany(e => e.Value) + .Where(p => filter(p.Value)) + .Select(p => new KeyValuePair(p.Value.Element.ElementDataKey + "." + p.Key, p.Value)); + }); IJsonBuilderElementData result = null; flowData.Setup(d => d.GetOrAdd( @@ -475,5 +521,25 @@ private bool ContainsJavaScriptProperties(string json) } return false; } + + private Type GetTypeFromEnum(JsPropertyType type) + { + Type result = typeof(JavaScript); + switch (type) + { + case JsPropertyType.JavaScript: + result = typeof(JavaScript); + break; + case JsPropertyType.IAspectPropertyValue: + result = typeof(IAspectPropertyValue); + break; + case JsPropertyType.AspectPropertyValue: + result = typeof(AspectPropertyValue); + break; + default: + break; + } + return result; + } } } diff --git a/FiftyOne.Pipeline.Engines/Utils.cs b/FiftyOne.Pipeline.Engines/Utils.cs new file mode 100644 index 00000000..1d9c87a4 --- /dev/null +++ b/FiftyOne.Pipeline.Engines/Utils.cs @@ -0,0 +1,42 @@ +using FiftyOne.Pipeline.Engines.Data; +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Engines +{ + /// + /// Static utility methods + /// + public static class Utils + { + /// + /// Check if the specified type is a specific type 'T' or an + /// implementation of + /// that wraps 'T'. + /// + /// + /// The type to check for + /// + /// + /// The type to check + /// + /// + /// True if 'type' is of type 'T' or a wrapper around 'T' + /// + /// + /// Thrown if required parameters are null + /// + public static bool IsTypeOrAspectPropertyValue(Type type) + { + if(type == null) + { + throw new ArgumentNullException(nameof(type)); + } + return type.Equals(typeof(T)) || + typeof(IAspectPropertyValue).IsAssignableFrom(type); + } + + + } +} diff --git a/FiftyOne.Pipeline.Web.Shared/Constants.cs b/FiftyOne.Pipeline.Web.Shared/Constants.cs new file mode 100644 index 00000000..84376262 --- /dev/null +++ b/FiftyOne.Pipeline.Web.Shared/Constants.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Web.Shared +{ + /// + /// Static class containing various constants that are used by the + /// Pipeline web integration and/or are helpful to callers. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", + "CA1707:Identifiers should not contain underscores", + Justification = "51Degrees coding style is for constant names " + + "to be all-caps with an underscore to separate words.")] + public static class Constants + { + /// + /// The Content type that indicates the body contains a + /// URL encoded form. + /// + public static readonly string[] CONTENT_TYPE_FORM = + { + "application/x-www-form-urlencoded", + "multipart/form-data" + }; + + /// + /// The HTTP method indicating this request was a POST. + /// + public const string METHOD_POST = "POST"; + } +} diff --git a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs index f19ec1fe..738499aa 100644 --- a/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs +++ b/Web Integration/FiftyOne.Pipeline.Web.Framework/WebPipeline.cs @@ -263,7 +263,19 @@ public static IFlowData Process(HttpRequest request) { CheckAndAdd(flowData, (string)sessionValueName, request.RequestContext.HttpContext.Session[(string)sessionValueName]); } + } + // Add form parameters to the evidence. + if (request.HttpMethod == Shared.Constants.METHOD_POST && + Shared.Constants.CONTENT_TYPE_FORM.Contains(request.ContentType)) + { + foreach (var formKey in request.Form.AllKeys) + { + string evidenceKey = Core.Constants.EVIDENCE_QUERY_PREFIX + + Core.Constants.EVIDENCE_SEPERATOR + formKey; + CheckAndAdd(flowData, evidenceKey, request.Form[formKey]); + } } + // Add the client IP CheckAndAdd(flowData, "server.client-ip", request.UserHostAddress); diff --git a/Web Integration/FiftyOne.Pipeline.Web/Services/WebRequestEvidenceService.cs b/Web Integration/FiftyOne.Pipeline.Web/Services/WebRequestEvidenceService.cs index 6c8367aa..31a1ccb0 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/Services/WebRequestEvidenceService.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/Services/WebRequestEvidenceService.cs @@ -22,6 +22,7 @@ using System; using System.Globalization; +using System.Linq; using FiftyOne.Pipeline.Core.Data; using Microsoft.AspNetCore.Http; @@ -126,7 +127,8 @@ public void AddEvidenceFromRequest(IFlowData flowData, HttpRequest httpRequest) CheckAndAdd(flowData, evidenceKey, queryValue.Value.ToString()); } // Add form parameters to the evidence. - if (httpRequest.Method == "POST") + if (httpRequest.Method == Shared.Constants.METHOD_POST && + Shared.Constants.CONTENT_TYPE_FORM.Contains(httpRequest.ContentType)) { foreach (var formValue in httpRequest.Form) { diff --git a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/WebRequestEvidenceServiceTest.cs b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/WebRequestEvidenceServiceTest.cs index cd88f766..f4c46991 100644 --- a/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/WebRequestEvidenceServiceTest.cs +++ b/Web Integration/Tests/FiftyOne.Pipeline.Web.Tests/WebRequestEvidenceServiceTest.cs @@ -73,16 +73,18 @@ IEnumerator IEnumerable.GetEnumerator() } - private static string expectedValue = "expected"; + private static string EXPECTED_VALUE = "expected"; - private static string requiredKey = "requiredkey"; - private static string notRequiredKey = "notrequiredkey"; + private static string REQUIRED_KEY = "requiredkey"; + private static string NOT_REQUIRED_KEY = "notrequiredkey"; + private static string FORM_KEY = "formValueKey"; + private static string FORM_VALUE = "formValueValue"; - private static IPAddress ip = new IPAddress(123321); + private static IPAddress IP = new IPAddress(123321); - private Mock flowData; - private WebRequestEvidenceService service; - private Mock request; + private Mock _flowData; + private WebRequestEvidenceService _service; + private Mock _request; /// /// Set up the test by creating a new service and initialising any @@ -92,26 +94,36 @@ IEnumerator IEnumerable.GetEnumerator() public void StartUp() { var values = new Dictionary() { - { requiredKey, expectedValue }, + { REQUIRED_KEY, EXPECTED_VALUE }, { "null", null } }; var valuesV = new Dictionary() { - { requiredKey, new StringValues(expectedValue) }, + { REQUIRED_KEY, new StringValues(EXPECTED_VALUE) }, { "null", new StringValues((string)null) } }; var headers = new HeaderDictionary(valuesV); var query = new QueryCollection(valuesV); var cookies = new MockCookieCollection(values); - - request = new Mock(); - flowData = new Mock(); - service = new WebRequestEvidenceService(); - - request.SetupGet(r => r.Headers).Returns(headers); - request.SetupGet(r => r.Cookies).Returns(cookies); - request.SetupGet(r => r.Query).Returns(query); - request.SetupGet(r => r.HttpContext.Connection.LocalIpAddress) - .Returns(ip); - request.SetupGet(r => r.IsHttps).Returns(true); + var formValues = new FormCollection( + new Dictionary() { + { FORM_KEY, new StringValues(FORM_VALUE) } + }); + + _request = new Mock(); + _flowData = new Mock(); + _service = new WebRequestEvidenceService(); + + _request.SetupGet(r => r.Headers).Returns(headers); + _request.SetupGet(r => r.Cookies).Returns(cookies); + _request.SetupGet(r => r.Query).Returns(query); + _request.SetupGet(r => r.Form).Returns(formValues); + + _request.SetupGet(r => r.HttpContext.Connection.LocalIpAddress) + .Returns(IP); + _request.SetupGet(r => r.IsHttps).Returns(true); + _request.SetupGet(r => r.ContentType) + .Returns(Shared.Constants.CONTENT_TYPE_FORM[0]); + _request.SetupGet(r => r.Method) + .Returns(Shared.Constants.METHOD_POST); } /// @@ -120,7 +132,7 @@ public void StartUp() /// required key to set private void SetRequiredKey(string key) { - flowData.SetupGet(f => f.EvidenceKeyFilter) + _flowData.SetupGet(f => f.EvidenceKeyFilter) .Returns(new EvidenceKeyFilterWhitelist( new List() { key })); } @@ -134,11 +146,11 @@ private void SetRequiredKey(string key) public void WebRequestEvidenceService_ContainsProtocol() { SetRequiredKey(Core.Constants.EVIDENCE_PROTOCOL); - service.AddEvidenceFromRequest(flowData.Object, request.Object); - flowData.Verify(f => f.AddEvidence( + _service.AddEvidenceFromRequest(_flowData.Object, _request.Object); + _flowData.Verify(f => f.AddEvidence( It.IsAny(), It.IsAny()), Times.Once); - flowData.Verify(f => f.AddEvidence( + _flowData.Verify(f => f.AddEvidence( Core.Constants.EVIDENCE_PROTOCOL, "https"), Times.Once); } @@ -152,15 +164,15 @@ public void WebRequestEvidenceService_ContainsProtocol() /// required key) is added to the flow data evidence collection. /// /// prefix to test - private void CheckRequired(string prefix) + private void CheckRequired(string prefix, string key, string expectedValue) { - SetRequiredKey(prefix + "." + requiredKey); - service.AddEvidenceFromRequest(flowData.Object, request.Object); - flowData.Verify(f => f.AddEvidence( + SetRequiredKey(prefix + "." + key); + _service.AddEvidenceFromRequest(_flowData.Object, _request.Object); + _flowData.Verify(f => f.AddEvidence( It.IsAny(), It.IsAny()), Times.Once); - flowData.Verify(f => f.AddEvidence( - prefix + "." + requiredKey, expectedValue), + _flowData.Verify(f => f.AddEvidence( + prefix + "." + key, expectedValue), Times.Once); } @@ -175,9 +187,9 @@ private void CheckRequired(string prefix) /// private void CheckNotRequired(string prefix) { - SetRequiredKey(prefix + "." + notRequiredKey); - service.AddEvidenceFromRequest(flowData.Object, request.Object); - flowData.Verify(f => f.AddEvidence( + SetRequiredKey(prefix + "." + NOT_REQUIRED_KEY); + _service.AddEvidenceFromRequest(_flowData.Object, _request.Object); + _flowData.Verify(f => f.AddEvidence( It.IsAny(), It.IsAny()), Times.Never); } @@ -191,11 +203,11 @@ public void WebRequestEvidenceService_AddClientIp() { var key = "server.client-ip"; SetRequiredKey(key); - service.AddEvidenceFromRequest(flowData.Object, request.Object); - flowData.Verify(f => f.AddEvidence( + _service.AddEvidenceFromRequest(_flowData.Object, _request.Object); + _flowData.Verify(f => f.AddEvidence( It.IsAny(), It.IsAny()), Times.Once); - flowData.Verify(f => f.AddEvidence(key, ip.ToString()), Times.Once); + _flowData.Verify(f => f.AddEvidence(key, IP.ToString()), Times.Once); } /// @@ -205,7 +217,7 @@ public void WebRequestEvidenceService_AddClientIp() [TestMethod] public void WebRequestEvidenceService_AddRequiredHeader() { - CheckRequired("header"); + CheckRequired("header", REQUIRED_KEY, EXPECTED_VALUE); } /// @@ -225,7 +237,7 @@ public void WebRequestEvidenceService_AddNotRequiredHeader() [TestMethod] public void WebRequestEvidenceService_AddRequiredCookie() { - CheckRequired("cookie"); + CheckRequired("cookie", REQUIRED_KEY, EXPECTED_VALUE); } /// @@ -245,7 +257,7 @@ public void WebRequestEvidenceService_AddNotRequiredCookie() [TestMethod] public void WebRequestEvidenceService_AddRequiredParam() { - CheckRequired("query"); + CheckRequired("query", REQUIRED_KEY, EXPECTED_VALUE); } /// @@ -259,6 +271,53 @@ public void WebRequestEvidenceService_AddNotRequiredParam() CheckNotRequired("query"); } + public static IEnumerable GetContentTypes + { + get + { + foreach (var entry in Shared.Constants.CONTENT_TYPE_FORM) + { + yield return new object[] { entry }; + } + } + } + /// + /// Test that evidence is added from form parameters for each + /// valid content type. + /// + [DataTestMethod] + [DynamicData(nameof(GetContentTypes))] + public void WebRequestEvidenceService_AddFormParam(string contentType) + { + _request.SetupGet(r => r.ContentType).Returns(contentType); + CheckRequired("query", FORM_KEY, FORM_VALUE); + } + + /// + /// Test that form parameters will not be read if type is not post + /// + [TestMethod] + public void WebRequestEvidenceService_AddFormParam_NotPost() + { + _request.SetupGet(r => r.Method).Returns("TEST"); + _request.SetupGet(r => r.Form).Throws(new System.Exception( + "This test should not be trying to access form values")); + CheckNotRequired("query"); + } + + /// + /// Test that form parameters will not be read if content type + /// is not url encoded form + /// + [TestMethod] + public void WebRequestEvidenceService_AddFormParam_NotForm() + { + _request.SetupGet(r => r.ContentType).Returns("TEST"); + _request.SetupGet(r => r.Form).Throws(new System.Exception( + "This test should not be trying to access form values")); + CheckNotRequired("query"); + } + /// /// Set the required key in the flow data to "prefix.null" where prefix /// if provided, and null is a null value which exists in headers, @@ -272,11 +331,11 @@ public void WebRequestEvidenceService_AddNotRequiredParam() private void CheckNullValue(string prefix) { SetRequiredKey(prefix + ".null"); - service.AddEvidenceFromRequest(flowData.Object, request.Object); - flowData.Verify(f => f.AddEvidence( + _service.AddEvidenceFromRequest(_flowData.Object, _request.Object); + _flowData.Verify(f => f.AddEvidence( It.IsAny(), It.IsAny()), Times.Once); - flowData.Verify(f => f.AddEvidence( + _flowData.Verify(f => f.AddEvidence( prefix + ".null", ""), Times.Once); } From 43d8689f40539c73a19503efc096fb4b4e059524 Mon Sep 17 00:00:00 2001 From: Steve Ballantine Date: Mon, 24 May 2021 12:25:26 +0000 Subject: [PATCH 10/14] BUG: Corrected a regression for a bug that was previously fixed. BUG: Corrected a regression for a bug that was previously fixed in 3da27c204c25e29c65d613c4478867625cd9f356 --- Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs index f271dd6d..102490f5 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneStartup.cs @@ -240,9 +240,9 @@ private static void AddJsElements(PipelineOptions options) { // There is already a JavaScript builder config so check if // the endpoint is specified. If not, add it. - if (jsonConfig.Single().BuildParameters.ContainsKey("EndPoint") == false) + if (javascriptConfig.Single().BuildParameters.ContainsKey("EndPoint") == false) { - jsonConfig.Single().BuildParameters.Add("EndPoint", "/51dpipeline/json"); + javascriptConfig.Single().BuildParameters.Add("EndPoint", "/51dpipeline/json"); } } } From 8cd5ffd76954297cc24d77aac10fd34b1caa4da5 Mon Sep 17 00:00:00 2001 From: Ben Shillito Date: Wed, 9 Jun 2021 09:15:35 +0000 Subject: [PATCH 11/14] Merged PR 4022: DOC: Commented the `UseFiftyOne` extension method to make it clear that it must be called before any `ExceptionHandlerExtensions` methods. DOC: Commented the `UseFiftyOne` extension method to make it clear that it must be called before any `ExceptionHandlerExtensions` methods. --- Web Integration/Examples/NetCore2.1/Startup.cs | 2 +- Web Integration/Examples/NetCore3.1/Startup.cs | 9 +++++---- .../FiftyOne.Pipeline.Web/FiftyOneExtensions.cs | 5 +++++ performance-tests/CMakeLists.txt | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Web Integration/Examples/NetCore2.1/Startup.cs b/Web Integration/Examples/NetCore2.1/Startup.cs index 7a22b43b..f9ffb1ac 100644 --- a/Web Integration/Examples/NetCore2.1/Startup.cs +++ b/Web Integration/Examples/NetCore2.1/Startup.cs @@ -143,11 +143,11 @@ public void ConfigureServices(IServiceCollection services) public void Configure(IApplicationBuilder app, IHostingEnvironment env) { - app.UseFiftyOne(); if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); } + app.UseFiftyOne(); app.UseMvc(routes => routes.MapRoute( name: "default", diff --git a/Web Integration/Examples/NetCore3.1/Startup.cs b/Web Integration/Examples/NetCore3.1/Startup.cs index db488f15..c7aa8206 100644 --- a/Web Integration/Examples/NetCore3.1/Startup.cs +++ b/Web Integration/Examples/NetCore3.1/Startup.cs @@ -63,10 +63,6 @@ public void ConfigureServices(IServiceCollection services) // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - // Call UseFiftyOne to add the Middleware component that will send any - // requests through the 51Degrees pipeline. - app.UseFiftyOne(); - if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); @@ -77,6 +73,11 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. app.UseHsts(); } + + // Call UseFiftyOne to add the Middleware component that will send any + // requests through the 51Degrees pipeline. + app.UseFiftyOne(); + app.UseHttpsRedirection(); app.UseStaticFiles(); diff --git a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneExtensions.cs b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneExtensions.cs index 74ce5668..9e0d81fb 100644 --- a/Web Integration/FiftyOne.Pipeline.Web/FiftyOneExtensions.cs +++ b/Web Integration/FiftyOne.Pipeline.Web/FiftyOneExtensions.cs @@ -39,6 +39,11 @@ public static class FiftyOneExtensions /// Directs the MVC ApplicationBuilder to use the 51Degrees middleware /// component to provide Pipeline functionality. /// + /// + /// Note: if any of the ExceptionHandlerExtensions.UseExceptionHandler + /// methods are used, they MUST be called before this method in order + /// to correctly handle exception. + /// /// The /// The public static IApplicationBuilder UseFiftyOne( diff --git a/performance-tests/CMakeLists.txt b/performance-tests/CMakeLists.txt index 79984cce..67f02cfc 100644 --- a/performance-tests/CMakeLists.txt +++ b/performance-tests/CMakeLists.txt @@ -8,7 +8,7 @@ set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) ExternalProject_Add(ApacheBench GIT_REPOSITORY https://github.com/51degrees/apachebench CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} - GIT_TAG origin/feature/wait-for-endpoint + GIT_TAG origin/main STEP_TARGETS build EXCLUDE_FROM_ALL TRUE ) From 90f385abf4044f92c9c4cdec9f34e7e55e078b25 Mon Sep 17 00:00:00 2001 From: Tung Pham Date: Thu, 10 Jun 2021 13:37:01 +0000 Subject: [PATCH 12/14] Merged PR 4056: Continuous Integration Changes made: - Moved all yaml files to 'ci' folder. - Added 'common-ci' as submodule - Added API CI/CD specific README.md - Created shared yaml template for build, test and create packages tasks. - Added yaml file for deployment pipeline. - Added state checks in pipeline script tasks. - Enabled failOnStderr option where relevant. --- .gitmodules | 5 +- build-and-test.yml | 133 ---------------- ci/README.md | 2 + ci/build-and-test.yml | 13 ++ ci/common-ci | 1 + ci/create-packages-debug.yml | 17 ++ ci/create-packages.yml | 27 ++++ ci/deploy.yml | 160 +++++++++++++++++++ ci/shared-build-and-test-stage.yml | 160 +++++++++++++++++++ ci/shared-build-test-create.yml | 21 +++ ci/shared-create-packages-stage.yml | 162 +++++++++++++++++++ ci/shared-extract-and-publish-packages.yml | 28 ++++ ci/shared-variables.yml | 5 + publish-packages.yml | 172 --------------------- 14 files changed, 600 insertions(+), 306 deletions(-) delete mode 100644 build-and-test.yml create mode 100644 ci/README.md create mode 100644 ci/build-and-test.yml create mode 160000 ci/common-ci create mode 100644 ci/create-packages-debug.yml create mode 100644 ci/create-packages.yml create mode 100644 ci/deploy.yml create mode 100644 ci/shared-build-and-test-stage.yml create mode 100644 ci/shared-build-test-create.yml create mode 100644 ci/shared-create-packages-stage.yml create mode 100644 ci/shared-extract-and-publish-packages.yml create mode 100644 ci/shared-variables.yml delete mode 100644 publish-packages.yml diff --git a/.gitmodules b/.gitmodules index 72faba1d..1eb32c81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElement/Templates/javascript"] path = FiftyOne.Pipeline.Elements/FiftyOne.Pipeline.JavaScriptBuilderElement/Templates/javascript - url=../javascript \ No newline at end of file + url=../javascript +[submodule "ci/common-ci"] + path = ci/common-ci +url=../common-ci \ No newline at end of file diff --git a/build-and-test.yml b/build-and-test.yml deleted file mode 100644 index 31f81963..00000000 --- a/build-and-test.yml +++ /dev/null @@ -1,133 +0,0 @@ -pool: - vmImage: 'windows-2019' - -trigger: none - -# Configure this to run for both Debug and Release configurations -strategy: - maxParallel: 4 - matrix: - debug: - BuildConfiguration: Debug - release: - BuildConfiguration: Release - -variables: - RestoreBuildProjects: '**/*.sln' - DOTNET_NOLOGO: true - -steps: -# Get the data files that are required for device detection automated system tests. -- powershell: | - git lfs install - ls - git config --global --add filter.lfs.required true - git config --global --add filter.lfs.smudge "git-lfs smudge -- %f" - git config --global --add filter.lfs.process "git-lfs filter-process" - git config --global --add filter.lfs.clean "git-lfs clean -- %f" - displayName: 'Configure git lfs' - -- checkout: self - lfs: true - submodules: recursive - -- task: NuGetToolInstaller@1 - displayName: 'Use NuGet 5.3.1' - inputs: - versionSpec: 5.3.1 - -- task: UseDotNet@2 - displayName: 'Use .NET Core 3.1' - inputs: - packageType: sdk - version: 3.1.x - performMultiLevelLookup: true - -- task: NuGetCommand@2 - displayName: 'NuGet restore' - inputs: - command: 'restore' - restoreSolution: '$(RestoreBuildProjects)' - feedsToUse: 'select' - vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' - -- task: VSBuild@1 - displayName: 'Build solutions' - inputs: - solution: '$(RestoreBuildProjects)' - vsVersion: '15.0' - platform: 'Any CPU' - configuration: '$(BuildConfiguration)' - clean: true - -- task: VisualStudioTestPlatformInstaller@1 - displayName: 'Visual Studio Test Platform Installer' - inputs: - versionSelector: latestStable - -- task: VSTest@2 - displayName: 'VsTest - testAssemblies - dotnet framework' - inputs: - testSelector: 'testAssemblies' - testAssemblyVer2: | - **\net4*\*Tests*.dll - !**\*TestAdapter*.dll - !**\*TestFramework*.dll - !**\obj\** - searchFolder: '$(System.DefaultWorkingDirectory)' - codeCoverageEnabled: true - otherConsoleOptions: '/Framework:Framework45 /logger:console;verbosity="normal"' - configuration: '$(BuildConfiguration)' - diagnosticsEnabled: true - testRunTitle: 'framework-$(BuildConfiguration)' - -- task: VSTest@2 - displayName: 'VsTest - testAssemblies - dotnet core' - inputs: - testSelector: 'testAssemblies' - testAssemblyVer2: | - **\netcoreapp*\*Tests*.dll - !**\*TestAdapter*.dll - !**\*TestFramework*.dll - !**\obj\** - !**\performance_tests.dll - searchFolder: '$(System.DefaultWorkingDirectory)' - codeCoverageEnabled: true - otherConsoleOptions: '/Framework:.NETCoreApp,Version=v3.1 /logger:console;verbosity="normal"' - configuration: '$(BuildConfiguration)' - diagnosticsEnabled: true - testRunTitle: 'dotnetcore-$(BuildConfiguration)' - -- task: PowerShell@2 - displayName: 'Build cloud perf tests' - inputs: - targetType: 'inline' - script: | - cd performance-tests/ - mkdir build - cd build - cmake .. - cmake --build . - cd .. - -- task: PowerShell@2 - displayName: 'Run cloud perf tests' - inputs: - filePath: '$(System.DefaultWorkingDirectory)/performance-tests/build/runPerf.ps1' - workingDirectory: '$(System.DefaultWorkingDirectory)' - -- task: DotNetCoreCLI@2 - displayName: 'Micro benchmarks' - inputs: - command: run - projects: '**/FiftyOne.Pipeline.Benchmarks.csproj' - nobuild: true - -- task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: 'performance-tests/build' - ArtifactName: 'perfout' - publishLocation: 'Container' - displayName: 'Publish Performance Artifacts' -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact' \ No newline at end of file diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..d58d90d0 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,2 @@ +# API Specific CI/CD Approach +This API complies with the `common-ci` approach. \ No newline at end of file diff --git a/ci/build-and-test.yml b/ci/build-and-test.yml new file mode 100644 index 00000000..e7f7252b --- /dev/null +++ b/ci/build-and-test.yml @@ -0,0 +1,13 @@ +# No push trigger. However, by default without PR trigger specified, +# this is run when a PR is created. +trigger: none + +# Use shared variables +variables: +- template: shared-variables.yml + +stages: +- template: shared-build-and-test-stage.yml + parameters: + imageName: $(imageName) + nugetVersion: $(nugetVersion) \ No newline at end of file diff --git a/ci/common-ci b/ci/common-ci new file mode 160000 index 00000000..55c6d402 --- /dev/null +++ b/ci/common-ci @@ -0,0 +1 @@ +Subproject commit 55c6d4024c63546b69a57980c6b13191968ae3c1 diff --git a/ci/create-packages-debug.yml b/ci/create-packages-debug.yml new file mode 100644 index 00000000..afaa6ed8 --- /dev/null +++ b/ci/create-packages-debug.yml @@ -0,0 +1,17 @@ +# Debug build is only needed for investigating issues so should be manually triggered only +trigger: none + +# Don't trigger for a pull request. +pr: none + +# Use shared variables +variables: +- template: shared-variables.yml + +# Perform build, test and create packages +stages: +- template: shared-build-test-create.yml + parameters: + targetPublishConfig: 'Debug' # Specify the target build configuration to publish. + nugetVersion: $(nugetVersion) + imageName: $(imageName) diff --git a/ci/create-packages.yml b/ci/create-packages.yml new file mode 100644 index 00000000..c3b1ab7f --- /dev/null +++ b/ci/create-packages.yml @@ -0,0 +1,27 @@ +trigger: + - master + - develop + - release/* + +# Don't trigger for a pull request +pr: none + +# Schedule to run overnight +schedules: +- cron: "0 20 * * *" + displayName: Daily overnight build + branches: + include: + - develop + +# Use shared variables +variables: +- template: shared-variables.yml + +# Perform Build, Test and Create +stages: +- template: shared-build-test-create.yml + parameters: + targetPublishConfig: 'Release' # Specify the target build configuration to publish + nugetVersion: $(nugetVersion) + imageName: $(imageName) \ No newline at end of file diff --git a/ci/deploy.yml b/ci/deploy.yml new file mode 100644 index 00000000..37c280e4 --- /dev/null +++ b/ci/deploy.yml @@ -0,0 +1,160 @@ +# Disable automatic runs of this pipeline when changes are pushed to the repository. +trigger: none + +# Disable automatic runs of this pipeline when a PR is create. +pr: none + +# Include the shared variables. +variables: +- template: shared-variables.yml + +# Add the pipeline that builds the packages as a resource. +# This allows the deployment pipeline to be triggered whenever +# the build pipeline completes. +resources: + pipelines: + - pipeline: build-pipeline # The name for the triggering build pipeline within this script + source: pipeline-dotnet-create-packages # Name of the pipeline from here: https://51degrees.visualstudio.com/Pipeline/_build + trigger: true + +stages: +# This should be triggered automatically. +- stage: publish_internal + displayName: Publish Internal + jobs: + # Note - A 'deployment' job will automatically download the artifacts created by the triggering pipeline. + - deployment: deploy_internal + displayName: Deploy Internal + pool: + vmImage: 'ubuntu-18.04' + workspace: + clean: all + environment: packages-internal + strategy: + runOnce: + deploy: + steps: + #- bash: 'ls -R "$(PIPELINE.WORKSPACE)"' + # displayName: 'List files and directories' + - template: shared-extract-and-publish-packages.yml + parameters: + nugetFeedType: 'internal' + nugetVersion: $(nugetVersion) + +# This stage is only triggered if done from a master branch and is approved by +# a list of approvers specified in the environment. +- stage: publish_nuget + displayName: Publish to NuGet.org + dependsOn: [] + condition: eq(variables['build.sourceBranch'], 'refs/heads/master') + jobs: + - deployment: deploy_nuget + displayName: Deploy to NuGet + pool: + vmImage: 'ubuntu-18.04' + workspace: + clean: all + environment: nuget + strategy: + runOnce: + deploy: + steps: + + - template: shared-extract-and-publish-packages.yml + parameters: + nugetFeedType: 'external' + nugetVersion: $(nugetVersion) + +- stage: publish_github + displayName: Publish to GitHub + dependsOn: publish_nuget + jobs: + - deployment: deploy_github + displayName: Deploy to GitHub + pool: + vmImage: 'ubuntu-18.04' + workspace: + clean: all + environment: github + strategy: + runOnce: + deploy: + steps: + + - task: DownloadSecureFile@1 + displayName: 'Download ssh private key' + name: githubsshkey + inputs: + secureFile: 'gihhub-ssh-key' + retryCount: 5 + steps: + + - task: DownloadSecureFile@1 + displayName: 'Download ssh public key' + name: githubsshkeypublic + inputs: + secureFile: 'gihhub-ssh-key-public' + retryCount: 5 + + - bash: | + # Copy ssh keys to expected location + mkdir ~/.ssh + mv $(githubsshkey.secureFilePath) ~/.ssh/id_rsa + mv $(githubsshkeypublic.secureFilePath) ~/.ssh/id_rsa.pub + if [ ! -f ~/.ssh/id_rsa ] && [ ! -f ~/.ssh/id_rsa ]; then + echo "Failed to obtain ssh keys." + exit 1 + fi + + # Have to change permissions on key files in order for SSH to work with them. + chmod 600 ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa.pub + export id_rsa_perm=`stat -c '%a' ~/.ssh/id_rsa` + export id_rsa_pub_perm=`stat -c '%a' ~/.ssh/id_rsa.pub` + if [ "$id_rsa_perm" != "600" ] && [ "$id_rsa_pub_perm" != "600" ]; then + echo "Failed to change permission on key files." + exit 1 + fi + + # Uncomment the line below to debug issues with SSH + #ssh -vT git@github.com + + echo "Cloning git repository" + git clone https://a:$(System.AccessToken)@dev.azure.com/51degrees/$(System.TeamProject)/_git/$(Build.Repository.Name) source + if [ ! -d source ]; then + echo "Failed to checkout repository." + exit 1 + fi + + cd source + + echo "Checking out branch '$(Build.SourceBranchName)'" + git checkout $(Build.SourceBranchName) + if [ $? != 0 ]; then + echo "Failed to checkout branch '$(Build.SourceBranchName)'." + exit 1 + fi + + echo "Adding GitHub remote" + git remote add github git@github.com:51Degrees/$(Build.Repository.Name).git + if [ $? != 0 ]; then + echo "Failed to add GitHub remote." + exit 1 + fi + + echo "Pushing branch '$(Build.SourceBranchName)' to GitHub" + git push github $(Build.SourceBranchName) + if [ $? != 0 ]; then + echo "Failed to push branch '$(Build.SourceBranchName)' to GitHub." + exit 1 + fi + + echo "Pushing tag '$(Build.BuildNumber)' to GitHub" + git push github $(Build.BuildNumber) + if [ $? != 0 ]; then + echo "Failed to push tag '$(Build.BuildNumber)' to GitHub." + exit 1 + fi + + displayName: 'Push to GitHub' + failOnStderr: true \ No newline at end of file diff --git a/ci/shared-build-and-test-stage.yml b/ci/shared-build-and-test-stage.yml new file mode 100644 index 00000000..d7d898ce --- /dev/null +++ b/ci/shared-build-and-test-stage.yml @@ -0,0 +1,160 @@ +parameters: +- name: imageName # Name of the agent to work on. +- name: nugetVersion + type: string + default: 5.8.0 + +stages: +- stage: Build_and_Test + + jobs: + - job: Build_and_Test + displayName: Build and Test + + pool: + vmImage: ${{ parameters.imageName }} + + # Configure this to run for both Debug and Release configurations + strategy: + maxParallel: 4 + matrix: + debug: + BuildConfiguration: Debug + release: + BuildConfiguration: Release + + variables: + RestoreBuildProjects: '**/*.sln' + DOTNET_NOLOGO: true + + steps: + # Get the data files that are required for device detection automated system tests. + - powershell: | + git lfs install + if (-Not $?) { + "ERROR: Failed to install git lft." + exit 1 + } + ls + + git config --global --add filter.lfs.required true + git config --global --add filter.lfs.smudge "git-lfs smudge -- %f" + git config --global --add filter.lfs.process "git-lfs filter-process" + git config --global --add filter.lfs.clean "git-lfs clean -- %f" + displayName: 'Configure git lfs' + failOnStderr: true + + - checkout: self + lfs: true + submodules: recursive + + - task: NuGetToolInstaller@1 + displayName: 'Use NuGet ${{ parameters.nugetVersion }}' + inputs: + versionSpec: ${{ parameters.nugetVersion }} + + - task: UseDotNet@2 + displayName: 'Use .NET Core 3.1' + inputs: + packageType: sdk + version: 3.1.x + performMultiLevelLookup: true + + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + command: 'restore' + restoreSolution: '$(RestoreBuildProjects)' + feedsToUse: 'select' + vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' + + - task: VSBuild@1 + displayName: 'Build solutions' + inputs: + solution: '$(RestoreBuildProjects)' + vsVersion: '15.0' + platform: 'Any CPU' + configuration: '$(BuildConfiguration)' + clean: true + + - task: VisualStudioTestPlatformInstaller@1 + displayName: 'Visual Studio Test Platform Installer' + inputs: + versionSelector: latestStable + + - task: VSTest@2 + displayName: 'VsTest - testAssemblies - dotnet framework' + inputs: + testSelector: 'testAssemblies' + testAssemblyVer2: | + **\net4*\*Tests*.dll + !**\*TestAdapter*.dll + !**\*TestFramework*.dll + !**\obj\** + searchFolder: '$(System.DefaultWorkingDirectory)' + codeCoverageEnabled: true + otherConsoleOptions: '/Framework:Framework45 /logger:console;verbosity="normal"' + configuration: '$(BuildConfiguration)' + diagnosticsEnabled: true + testRunTitle: 'framework-$(BuildConfiguration)' + + - task: VSTest@2 + displayName: 'VsTest - testAssemblies - dotnet core' + inputs: + testSelector: 'testAssemblies' + testAssemblyVer2: | + **\netcoreapp*\*Tests*.dll + !**\*TestAdapter*.dll + !**\*TestFramework*.dll + !**\obj\** + !**\performance_tests.dll + searchFolder: '$(System.DefaultWorkingDirectory)' + codeCoverageEnabled: true + otherConsoleOptions: '/Framework:.NETCoreApp,Version=v3.1 /logger:console;verbosity="normal"' + configuration: '$(BuildConfiguration)' + diagnosticsEnabled: true + testRunTitle: 'dotnetcore-$(BuildConfiguration)' + + - task: PowerShell@2 + displayName: 'Build cloud perf tests' + inputs: + targetType: 'inline' + script: | + cd performance-tests/ + mkdir build + if (-Not (Test-Path -Path ../performance-tests/build)) { + "ERROR: Failed to create build folder!" + exit 1 + } + + cd build + cmake .. + cmake --build . + if (-Not $?) { + "ERROR: Failed to build the performance test!" + exit 1 + } + cd .. + failOnStderr: true + + - task: PowerShell@2 + displayName: 'Run cloud perf tests' + inputs: + filePath: '$(System.DefaultWorkingDirectory)/performance-tests/build/runPerf.ps1' + workingDirectory: '$(System.DefaultWorkingDirectory)' + + - task: DotNetCoreCLI@2 + displayName: 'Micro benchmarks' + inputs: + command: run + projects: '**/FiftyOne.Pipeline.Benchmarks.csproj' + nobuild: true + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: 'performance-tests/build' + ArtifactName: 'perfout' + publishLocation: 'Container' + displayName: 'Publish Performance Artifacts' + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' \ No newline at end of file diff --git a/ci/shared-build-test-create.yml b/ci/shared-build-test-create.yml new file mode 100644 index 00000000..1361cc0c --- /dev/null +++ b/ci/shared-build-test-create.yml @@ -0,0 +1,21 @@ +parameters: +- name: targetPublishConfig # The target build configuration to publish + type: string + default: 'Release' +- name: nugetVersion # The NuGet version to use +- name: imageName # The agent to work on. + +stages: +# Build and Test the current source code +- template: shared-build-and-test-stage.yml + parameters: + nugetVersion: ${{ parameters.nugetVersion }} + imageName: ${{ parameters.imageName }} + +# Build source and create packages +- template: shared-create-packages-stage.yml + parameters: + dependency: Build_and_Test # Name of the build and test stage + nugetVersion: ${{ parameters.nugetVersion }} + imageName: ${{ parameters.imageName }} + targetPublishConfig: ${{ parameters.targetPublishConfig }} \ No newline at end of file diff --git a/ci/shared-create-packages-stage.yml b/ci/shared-create-packages-stage.yml new file mode 100644 index 00000000..393237b7 --- /dev/null +++ b/ci/shared-create-packages-stage.yml @@ -0,0 +1,162 @@ +parameters: +- name: targetPublishConfig # Target build configuration to publish + type: string + default: 'Release' +- name: nugetVersion # NuGet version to use + type: string + default: 5.8.0 +- name: imageName # Name of the agent to work on +- name: dependency # Name of the stage that this stage depends on + +stages: +- stage: CreatePackages + dependsOn: ${{ parameters.dependency }} + + variables: + - group: CertificateVariables + # Because we are pulling in a group, we need to define local variables + # using the name/value syntax. + - name: RestoreBuildProjects + value: '**/*.sln' + # Projects to be published as NuGet packages. + # Note the the Web and Web.Framework projects are published as a single package + # using a nuspec file rather than directly from the project files. Hence they + # are excluded here. + - name: PublishProjects + value: '**/*.csproj;!**/*[Tt]ests/**/*.csproj;!**/*[Ee]xamples/**/*.csproj;!**/FiftyOne.Pipeline.Web.csproj;!**/FiftyOne.Pipeline.Web.Framework.csproj' + # Access token for the git repository. Used by the git tag task. + - name: system_accesstoken + value: $(System.AccessToken) + + jobs: + - job: CreatePackages + displayName: Create Packages + + pool: + vmImage: ${{ parameters.imageName }} + + steps: + - bash: | + git lfs install + if [ $? != 0 ]; then + echo "ERROR: Failed to install lfs." + fi + ls + + git config --global --add filter.lfs.required true + git config --global --add filter.lfs.smudge "git-lfs smudge -- %f" + git config --global --add filter.lfs.process "git-lfs filter-process" + git config --global --add filter.lfs.clean "git-lfs clean -- %f" + displayName: 'Configure git lfs' + failOnStderr: true + + # The lines below are needed to allow the pipeline access to the + # OAuth access token that controls write access to the git repository. + # (Required for GitTag task) + - checkout: self + lfs: true + submodules: recursive + persistCredentials: true + # Useful snippets for debugging. + # List all contents of a directory: + #- script: | + # ls -d $(System.ArtifactsDirectory)/* + + - task: NuGetToolInstaller@1 + displayName: 'Use NuGet ${{ parameters.nugetVersion }}' + inputs: + versionSpec: ${{ parameters.nugetVersion }} + + - task: UseDotNet@2 + displayName: 'Use .NET Core 3.1' + inputs: + packageType: sdk + version: 3.1.x + performMultiLevelLookup: true + + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + command: 'restore' + restoreSolution: '$(RestoreBuildProjects)' + feedsToUse: 'select' + vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' + + - task: gittools.gitversion.gitversion-task.GitVersion@5 + displayName: 'Determine Version Number' + # Give this task a name so we can use the variables it sets later. + name: GitVersion + inputs: + preferBundledVersion: false + + - task: VSBuild@1 + displayName: 'Build solutions Any CPU' + inputs: + solution: '$(RestoreBuildProjects)' + vsVersion: '15.0' + platform: 'Any CPU' + configuration: '${{ parameters.targetPublishConfig }}' + clean: true + msbuildArchitecture: 'x86' + + # Index and publish symbol file to allow debugging. + - task: PublishSymbols@2 + displayName: 'Publish Symbols' + inputs: + SearchPattern: '**/bin/**/*.pdb' + SymbolServerType: 'TeamServices' + SymbolsVersion: '$(GitVersion.NuGetVersion)' + condition: and(succeeded(), eq('${{ parameters.targetPublishConfig }}', 'Debug')) + + # The nuget package version uses the BUILD_BUILDNUMER environment variable. + # This has been set by the GitVersion task above. + - task: DotNetCoreCLI@2 + displayName: 'Build NuGet Package' + inputs: + command: 'pack' + packagesToPack: '$(PublishProjects)' + versioningScheme: 'byEnvVar' + versionEnvVar: 'BUILD_BUILDNUMBER' + + # The Web and Web.Framework projects are combined into a single NuGet package. + # This requires the use of a nuspec file and the NuGet task. + - task: NuGetCommand@2 + displayName: 'NuGet pack Pipeline.Web' + inputs: + command: 'pack' + packagesToPack: '**/FiftyOne.Pipeline.Web.nuspec' + versioningScheme: 'byEnvVar' + versionEnvVar: 'BUILD_BUILDNUMBER' + buildProperties: 'config=${{ parameters.targetPublishConfig }}' + + # The secure file to download will be stored in the + # Pipelines/Library/SecureFiles section in Azure DevOps. + - task: DownloadSecureFile@1 + displayName: 'Download Code Signing Certificate' + name: CodeSigningCert + inputs: + secureFile: '51Degrees.mobi Code Signing Certificate.pfx' + + # Sign the Nuget package with the file downloaded previously. + # The password is stored in the Pipelines/Library/VariableGroups + # section in Azure DevOps. + - task: NuGetCommand@2 + displayName: 'Sign NuGet Package' + inputs: + command: custom + arguments: 'sign $(System.ArtifactsDirectory)\*.nupkg -CertificatePath "$(CodeSigningCert.secureFilePath)" -CertificatePassword $(CodeSigningCertPassword) -Timestamper http://timestamp.digicert.com' + + # Add a tag to the git repository with the version number of + # the package that has just been published + - task: ATP.ATP-GitTag.GitTag.GitTag@5 + displayName: 'Tag Repo With Version Number' + inputs: + tagUser: 'Azure DevOps' + tagEmail: 'CIUser@51Degrees.com' + condition: and(succeeded(), eq('${{ parameters.targetPublishConfig }}', 'Release')) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact' + inputs: + PathtoPublish: '$(build.artifactstagingdirectory)' + condition: succeededOrFailed() \ No newline at end of file diff --git a/ci/shared-extract-and-publish-packages.yml b/ci/shared-extract-and-publish-packages.yml new file mode 100644 index 00000000..357d110d --- /dev/null +++ b/ci/shared-extract-and-publish-packages.yml @@ -0,0 +1,28 @@ +parameters: +- name: nugetVersion +- name: nugetFeedType + +steps: + +# Install NuGet and restore packages +- task: NuGetToolInstaller@1 + displayName: 'Use NuGet ${{ parameters.nugetVersion }}' + inputs: + versionSpec: ${{ parameters.nugetVersion }} + +- task: NuGetCommand@2 + displayName: 'NuGet publish internal' + inputs: + command: push + packagesToPush: '$(PIPELINE.WORKSPACE)/build-pipeline/drop/**/*.nupkg;!$(PIPELINE.WORKSPACE)/build-pipeline/drop/**/*.symbols.nupkg' + publishVstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' + condition: and(succeeded(), eq('${{ parameters.nugetFeedType }}', 'internal')) + +- task: NuGetCommand@2 + displayName: 'NuGet publish external' + inputs: + command: push + packagesToPush: '$(PIPELINE.WORKSPACE)/build-pipeline/drop/**/*.nupkg;!$(PIPELINE.WORKSPACE)/build-pipeline/drop/**/*.symbols.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'NuGet.org - Release' + condition: and(succeeded(), eq('${{ parameters.nugetFeedType }}', 'external')) \ No newline at end of file diff --git a/ci/shared-variables.yml b/ci/shared-variables.yml new file mode 100644 index 00000000..ec159f08 --- /dev/null +++ b/ci/shared-variables.yml @@ -0,0 +1,5 @@ +variables: +- name: nugetVersion # NuGet version that we should use across stages + value: 5.8.0 +- name: imageName # The agent that we support for .NET + value: windows-2019 \ No newline at end of file diff --git a/publish-packages.yml b/publish-packages.yml deleted file mode 100644 index 59fd4c31..00000000 --- a/publish-packages.yml +++ /dev/null @@ -1,172 +0,0 @@ -trigger: - - master - - develop - - release/* - -variables: - - group: CertificateVariables - # Because we are pulling in a group, we need to define local variables - # using the name/value syntax. - - name: RestoreBuildProjects - value: '**/*.sln' - # Projects to be published as NuGet packages. - # Note the the Web and Web.Framework projects are published as a single package - # using a nuspec file rather than directly from the project files. Hence they - # are excluded here. - - name: PublishProjects - value: '**/*.csproj;!**/*[Tt]ests/**/*.csproj;!**/*[Ee]xamples/**/*.csproj;!**/FiftyOne.Pipeline.Web.csproj;!**/FiftyOne.Pipeline.Web.Framework.csproj' - # Access token for the git repository. Used by the git tag task. - - name: system_accesstoken - value: $(System.AccessToken) - -pool: - vmImage: 'windows-2019' - -steps: -- bash: | - git lfs install - ls - git config --global --add filter.lfs.required true - git config --global --add filter.lfs.smudge "git-lfs smudge -- %f" - git config --global --add filter.lfs.process "git-lfs filter-process" - git config --global --add filter.lfs.clean "git-lfs clean -- %f" - displayName: 'Configure git lfs' - -# The lines below are needed to allow the pipeline access to the -# OAuth access token that controls write access to the git repository. -# (Required for GitTag task) -- checkout: self - lfs: true - submodules: recursive - persistCredentials: true -# Useful snippets for debugging. -# List all contents of a directory: -#- script: | -# ls -d $(System.ArtifactsDirectory)/* - -# Use a script to set the BuildConfiguration variable. -# If building from master or a release branch then build -# the Release configuration. Otherwise, build the Debug configuration. -- script: | - echo Current branch is "%BUILD_SOURCEBRANCH%" - echo %BUILD_SOURCEBRANCH% |findstr /b "refs/heads/release/*" > nul && ( - set ISRELEASEBRANCH=true - ) || ( - set ISRELEASEBRANCH=false - ) - if "%BUILD_SOURCEBRANCHNAME%" == "master" ( - SET BUILD_CONFIG=Release - ) else if %ISRELEASEBRANCH% == true ( - SET BUILD_CONFIG=Release - ) else ( - SET BUILD_CONFIG=Debug - ) - echo ##vso[task.setvariable variable=BuildConfiguration]%BUILD_CONFIG% - echo BuildConfiguration set to '%BUILD_CONFIG%' - displayName: 'Determine Build Configuration' - -- task: NuGetToolInstaller@1 - displayName: 'Use NuGet 5.3.1' - inputs: - versionSpec: 5.3.1 - -- task: UseDotNet@2 - displayName: 'Use .NET Core 3.1' - inputs: - packageType: sdk - version: 3.1.x - performMultiLevelLookup: true - -- task: NuGetCommand@2 - displayName: 'NuGet restore' - inputs: - command: 'restore' - restoreSolution: '$(RestoreBuildProjects)' - feedsToUse: 'select' - vstsFeed: 'd2431f86-c1e6-4d8b-8d27-311cf3614847' - -- task: gitversion/setup@0 - displayName: Install GitVersion - inputs: - versionSpec: '5.x' - -- task: gitversion/execute@0 - displayName: Determine Version - -- task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - downloadType: specific - itemPattern: '**/*' - downloadPath: '$(build.sourcesdirectory)/' - -- task: VSBuild@1 - displayName: 'Build solutions Any CPU' - inputs: - solution: '$(RestoreBuildProjects)' - vsVersion: '15.0' - platform: 'Any CPU' - configuration: '$(BuildConfiguration)' - clean: true - msbuildArchitecture: 'x86' - -# Index and publish symbol file to allow debugging. -- task: PublishSymbols@2 - displayName: 'Publish Symbols' - inputs: - SearchPattern: '**/bin/**/*.pdb' - SymbolServerType: 'TeamServices' - SymbolsVersion: '$(GitVersion.NuGetVersion)' - -# The nuget package version uses the BUILD_BUILDNUMER environment variable. -# This has been set by the GitVersion task above. -- task: DotNetCoreCLI@2 - displayName: 'Build NuGet Package' - inputs: - command: 'pack' - packagesToPack: '$(PublishProjects)' - versioningScheme: 'byEnvVar' - versionEnvVar: 'BUILD_BUILDNUMBER' - -# The Web and Web.Framework projects are combined into a single NuGet package. -# This requires the use of a nuspec file and the NuGet task. -- task: NuGetCommand@2 - displayName: 'NuGet pack Pipeline.Web' - inputs: - command: 'pack' - packagesToPack: '**/FiftyOne.Pipeline.Web.nuspec' - versioningScheme: 'byEnvVar' - versionEnvVar: 'BUILD_BUILDNUMBER' - buildProperties: 'config=$(BuildConfiguration)' - -# The secure file to download will be stored in the -# Pipelines/Library/SecureFiles section in Azure DevOps. -- task: DownloadSecureFile@1 - displayName: 'Download Code Signing Certificate' - name: CodeSigningCert - inputs: - secureFile: ' 51Degrees.mobi Code Signing Certificate.pfx' - -# Sign the Nuget package with the file downloaded previously. -# The password is stored in the Pipelines/Library/VariableGroups -# section in Azure DevOps. -- task: NuGetCommand@2 - displayName: 'Sign NuGet Package' - inputs: - command: custom - arguments: 'sign $(System.ArtifactsDirectory)\*.nupkg -CertificatePath "$(CodeSigningCert.secureFilePath)" -CertificatePassword $(CodeSigningCertPassword) -Timestamper http://timestamp.digicert.com' - -# Add a tag to the git repository with the version number of -# the package that has just been published -- task: ATP.ATP-GitTag.GitTag.GitTag@5 - displayName: 'Tag Repo With Version Number' - inputs: - tagUser: 'Azure DevOps' - tagEmail: 'CIUser@51Degrees.com' - condition: succeeded() - -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact' - inputs: - PathtoPublish: '$(build.artifactstagingdirectory)' - condition: succeededOrFailed() \ No newline at end of file From dad2015f69e1500c4b2f64d466e5436abe6e1bb1 Mon Sep 17 00:00:00 2001 From: Tung Pham Date: Tue, 15 Jun 2021 09:08:56 +0000 Subject: [PATCH 13/14] Merged PR 4083: Use common-ci template Changes made: - Use common-ci template - Updated doc version to 4.3 --- ci/common-ci | 2 +- ci/deploy.yml | 109 +++++----------------------------------- ci/shared-variables.yml | 4 +- docs/Doxyfile | 7 +-- docs/DoxygenLayout.xml | 2 +- 5 files changed, 22 insertions(+), 102 deletions(-) diff --git a/ci/common-ci b/ci/common-ci index 55c6d402..567c56af 160000 --- a/ci/common-ci +++ b/ci/common-ci @@ -1 +1 @@ -Subproject commit 55c6d4024c63546b69a57980c6b13191968ae3c1 +Subproject commit 567c56aff68566cbec378cf5d64ef78ab53910ca diff --git a/ci/deploy.yml b/ci/deploy.yml index 37c280e4..df6e29b2 100644 --- a/ci/deploy.yml +++ b/ci/deploy.yml @@ -7,6 +7,8 @@ pr: none # Include the shared variables. variables: - template: shared-variables.yml +- name: targetBranch + value: 'refs/heads/master' # Add the pipeline that builds the packages as a resource. # This allows the deployment pipeline to be triggered whenever @@ -16,6 +18,10 @@ resources: - pipeline: build-pipeline # The name for the triggering build pipeline within this script source: pipeline-dotnet-create-packages # Name of the pipeline from here: https://51degrees.visualstudio.com/Pipeline/_build trigger: true + repositories: + - repository: ciTemplates # Id of the repository used to reference to in this script + type: git + name: common-ci # Name of the actual repository stages: # This should be triggered automatically. @@ -26,7 +32,7 @@ stages: - deployment: deploy_internal displayName: Deploy Internal pool: - vmImage: 'ubuntu-18.04' + vmImage: $(linuxImage) workspace: clean: all environment: packages-internal @@ -46,12 +52,12 @@ stages: - stage: publish_nuget displayName: Publish to NuGet.org dependsOn: [] - condition: eq(variables['build.sourceBranch'], 'refs/heads/master') + condition: eq(variables['build.sourceBranch'], variables['targetBranch']) jobs: - deployment: deploy_nuget displayName: Deploy to NuGet pool: - vmImage: 'ubuntu-18.04' + vmImage: $(linuxImage) workspace: clean: all environment: nuget @@ -65,96 +71,7 @@ stages: nugetFeedType: 'external' nugetVersion: $(nugetVersion) -- stage: publish_github - displayName: Publish to GitHub - dependsOn: publish_nuget - jobs: - - deployment: deploy_github - displayName: Deploy to GitHub - pool: - vmImage: 'ubuntu-18.04' - workspace: - clean: all - environment: github - strategy: - runOnce: - deploy: - steps: - - - task: DownloadSecureFile@1 - displayName: 'Download ssh private key' - name: githubsshkey - inputs: - secureFile: 'gihhub-ssh-key' - retryCount: 5 - steps: - - - task: DownloadSecureFile@1 - displayName: 'Download ssh public key' - name: githubsshkeypublic - inputs: - secureFile: 'gihhub-ssh-key-public' - retryCount: 5 - - - bash: | - # Copy ssh keys to expected location - mkdir ~/.ssh - mv $(githubsshkey.secureFilePath) ~/.ssh/id_rsa - mv $(githubsshkeypublic.secureFilePath) ~/.ssh/id_rsa.pub - if [ ! -f ~/.ssh/id_rsa ] && [ ! -f ~/.ssh/id_rsa ]; then - echo "Failed to obtain ssh keys." - exit 1 - fi - - # Have to change permissions on key files in order for SSH to work with them. - chmod 600 ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa.pub - export id_rsa_perm=`stat -c '%a' ~/.ssh/id_rsa` - export id_rsa_pub_perm=`stat -c '%a' ~/.ssh/id_rsa.pub` - if [ "$id_rsa_perm" != "600" ] && [ "$id_rsa_pub_perm" != "600" ]; then - echo "Failed to change permission on key files." - exit 1 - fi - - # Uncomment the line below to debug issues with SSH - #ssh -vT git@github.com - - echo "Cloning git repository" - git clone https://a:$(System.AccessToken)@dev.azure.com/51degrees/$(System.TeamProject)/_git/$(Build.Repository.Name) source - if [ ! -d source ]; then - echo "Failed to checkout repository." - exit 1 - fi - - cd source - - echo "Checking out branch '$(Build.SourceBranchName)'" - git checkout $(Build.SourceBranchName) - if [ $? != 0 ]; then - echo "Failed to checkout branch '$(Build.SourceBranchName)'." - exit 1 - fi - - echo "Adding GitHub remote" - git remote add github git@github.com:51Degrees/$(Build.Repository.Name).git - if [ $? != 0 ]; then - echo "Failed to add GitHub remote." - exit 1 - fi - - echo "Pushing branch '$(Build.SourceBranchName)' to GitHub" - git push github $(Build.SourceBranchName) - if [ $? != 0 ]; then - echo "Failed to push branch '$(Build.SourceBranchName)' to GitHub." - exit 1 - fi - - echo "Pushing tag '$(Build.BuildNumber)' to GitHub" - git push github $(Build.BuildNumber) - if [ $? != 0 ]; then - echo "Failed to push tag '$(Build.BuildNumber)' to GitHub." - exit 1 - fi - - displayName: 'Push to GitHub' - failOnStderr: true \ No newline at end of file +- template: shared-publish-github-stage.yml@ciTemplates + parameters: + imageName: $(linuxImage) + branchName: ${{ variables.targetBranch }} \ No newline at end of file diff --git a/ci/shared-variables.yml b/ci/shared-variables.yml index ec159f08..8d3eda17 100644 --- a/ci/shared-variables.yml +++ b/ci/shared-variables.yml @@ -2,4 +2,6 @@ variables: - name: nugetVersion # NuGet version that we should use across stages value: 5.8.0 - name: imageName # The agent that we support for .NET - value: windows-2019 \ No newline at end of file + value: windows-2019 +- name: linuxImage # Name of the Linux imageName + value: ubuntu-18.04 \ No newline at end of file diff --git a/docs/Doxyfile b/docs/Doxyfile index 03260608..adb8bf0f 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "51Degrees Pipeline .NET" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 4.2 +PROJECT_NUMBER = 4.3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -56,7 +56,7 @@ PROJECT_LOGO = ../../../docs/images/logo-51Degrees-Docs.png # The PROJECT_URL tag is used to specify the URL which is linked by the # PROJECT_LOGO. -PROJECT_URL = ../../documentation/4.2/index.html +PROJECT_URL = ../../documentation/4.3/index.html # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is @@ -1106,7 +1106,7 @@ GENERATE_HTML = YES # The default directory is: html. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_OUTPUT = 4.2 +HTML_OUTPUT = 4.3 # The HTML_FILE_EXTENSION tag can be used to specify the file extension for each # generated HTML page (for example: .htm, .php, .asp). @@ -1183,6 +1183,7 @@ HTML_EXTRA_FILES = ../../../docs/images/icon-arrow-left-black.svg \ ../../../docs/images/icon-arrow-solid-right-mute.svg \ ../../../docs/images/icon-arrow-solid-right-orange.svg \ ../../../docs/images/icon-search.svg \ + ../../../docs/examplegrabber.js \ ../../../docs/search51.js # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen diff --git a/docs/DoxygenLayout.xml b/docs/DoxygenLayout.xml index 1d6f81a6..1fbe656c 100644 --- a/docs/DoxygenLayout.xml +++ b/docs/DoxygenLayout.xml @@ -2,7 +2,7 @@ - + From 75b48b9be81f16b1f468aa585fca4c1959fb8f99 Mon Sep 17 00:00:00 2001 From: Joseph Dix Date: Tue, 15 Jun 2021 10:18:11 +0000 Subject: [PATCH 14/14] Merged PR 4014: BUILD: Reference main branch in apache-bench repo for performance tests --- performance-tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-tests/CMakeLists.txt b/performance-tests/CMakeLists.txt index 67f02cfc..554a9f8b 100644 --- a/performance-tests/CMakeLists.txt +++ b/performance-tests/CMakeLists.txt @@ -8,7 +8,7 @@ set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external) ExternalProject_Add(ApacheBench GIT_REPOSITORY https://github.com/51degrees/apachebench CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} - GIT_TAG origin/main + GIT_TAG main STEP_TARGETS build EXCLUDE_FROM_ALL TRUE )