diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 4267554880..375dd34105 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
-
+
https://github.com/dotnet/arcade
- 85f3aa16d8797b5020f1fda11df1a958feb5f8df
+ 685008547318b269dbbb82cfc9ca968126301bbe
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index 18823840b1..4ed5c54e73 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0
. $PSScriptRoot\tools.ps1
# Add source entry to PackageSources
-function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
if ($packageSource -eq $null)
@@ -49,11 +49,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern
Write-Host "Package source $SourceName already present."
}
- AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+ AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd
}
# Add a credential node for the specified source
-function AddCredential($creds, $source, $username, $password) {
+function AddCredential($creds, $source, $username, $pwd) {
# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
@@ -82,17 +82,17 @@ function AddCredential($creds, $source, $username, $password) {
$passwordElement.SetAttribute("key", "ClearTextPassword")
$sourceElement.AppendChild($passwordElement) | Out-Null
}
- $passwordElement.SetAttribute("value", $Password)
+ $passwordElement.SetAttribute("value", $pwd)
}
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) {
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
ForEach ($PackageSource in $maestroPrivateSources) {
Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
- AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+ AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd
}
}
@@ -144,24 +144,24 @@ if ($disabledSources -ne $null) {
$userName = "dn-bot"
# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}
$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']")
if ($dotnet5Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
- AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}
$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
if ($dotnet6Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
- AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
}
$doc.Save($filename)
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 8943da242f..33a6f2d0e2 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -26,6 +26,7 @@ Param(
[string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '',
[switch] $excludePrereleaseVS,
+ [switch] $nativeToolsOnMachine,
[switch] $help,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
@@ -67,6 +68,7 @@ function Print-Usage() {
Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
+ Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -146,6 +148,9 @@ try {
$nodeReuse = $false
}
+ if ($nativeToolsOnMachine) {
+ $env:NativeToolsOnMachine = $true
+ }
if ($restore) {
InitializeNativeTools
}
diff --git a/eng/common/build.sh b/eng/common/build.sh
index bc07a1c684..a16e18b174 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -188,9 +188,6 @@ function InitializeCustomToolset {
function Build {
- if [[ "$ci" == true ]]; then
- TryLogClientIpAddress
- fi
InitializeToolset
InitializeCustomToolset
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index ec8971eb01..51f30e53dd 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -1,5 +1,12 @@
set(CROSS_ROOTFS $ENV{ROOTFS_DIR})
+# reset platform variables (e.g. cmake 3.25 sets LINUX=1)
+unset(LINUX)
+unset(FREEBSD)
+unset(ILLUMOS)
+unset(ANDROID)
+unset(TIZEN)
+
set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
set(CMAKE_SYSTEM_NAME FreeBSD)
diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1
index 435e764134..8fda30bdce 100644
--- a/eng/common/darc-init.ps1
+++ b/eng/common/darc-init.ps1
@@ -1,6 +1,6 @@
param (
$darcVersion = $null,
- $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+ $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
$verbosity = 'minimal',
$toolpath = $null
)
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index 39abdbecdc..4e4116f1d0 100644
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -2,7 +2,7 @@
source="${BASH_SOURCE[0]}"
darcVersion=''
-versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
verbosity='minimal'
while [[ $# > 0 ]]; do
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
index fdfeea66e7..b09ea669f9 100755
--- a/eng/common/dotnet-install.sh
+++ b/eng/common/dotnet-install.sh
@@ -52,7 +52,7 @@ done
# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
cpuname=$(uname -m)
case $cpuname in
- aarch64)
+ arm64|aarch64)
buildarch=arm64
;;
amd64|x86_64)
@@ -61,7 +61,7 @@ case $cpuname in
armv*l)
buildarch=arm
;;
- i686)
+ i[3-6]86)
buildarch=x86
;;
*)
diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1
deleted file mode 100644
index 0728b1a8b5..0000000000
--- a/eng/common/generate-graph-files.ps1
+++ /dev/null
@@ -1,86 +0,0 @@
-Param(
- [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
- [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
- [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
- [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created
- [string] $darcVersion, # darc's version
- [string] $graphvizVersion = '2.38', # GraphViz version
- [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
- # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
-)
-
-function CheckExitCode ([string]$stage)
-{
- $exitCode = $LASTEXITCODE
- if ($exitCode -ne 0) {
- Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
- ExitWithExitCode $exitCode
- }
-}
-
-try {
- $ErrorActionPreference = 'Stop'
- . $PSScriptRoot\tools.ps1
-
- Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
-
- Push-Location $PSScriptRoot
-
- Write-Host 'Installing darc...'
- . .\darc-init.ps1 -darcVersion $darcVersion
- CheckExitCode 'Running darc-init'
-
- $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
- $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
- $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
- $installBin = Join-Path $graphvizInstallDir 'bin'
-
- Write-Host 'Installing dot...'
- .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
-
- $darcExe = "$env:USERPROFILE\.dotnet\tools"
- $darcExe = Resolve-Path "$darcExe\darc.exe"
-
- Create-Directory $outputFolder
-
- # Generate 3 graph descriptions:
- # 1. Flat with coherency information
- # 2. Graphviz (dot) file
- # 3. Standard dependency graph
- $graphVizFilePath = "$outputFolder\graphviz.txt"
- $graphVizImageFilePath = "$outputFolder\graph.png"
- $normalGraphFilePath = "$outputFolder\graph-full.txt"
- $flatGraphFilePath = "$outputFolder\graph-flat.txt"
- $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
-
- if ($includeToolset) {
- Write-Host 'Toolsets will be included in the graph...'
- $baseOptions += @( '--include-toolset' )
- }
-
- Write-Host 'Generating standard dependency graph...'
- & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
- CheckExitCode 'Generating normal dependency graph'
-
- Write-Host 'Generating flat dependency graph and graphviz file...'
- & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
- CheckExitCode 'Generating flat and graphviz dependency graph'
-
- Write-Host "Generating graph image $graphVizFilePath"
- $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
- & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
- CheckExitCode 'Generating graphviz image'
-
- Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
-}
-catch {
- if (!$includeToolset) {
- Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
- Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
- }
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Category 'Arcade' -Message $_
- ExitWithExitCode 1
-} finally {
- Pop-Location
-}
\ No newline at end of file
diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1
new file mode 100644
index 0000000000..3e5c1c74a1
--- /dev/null
+++ b/eng/common/generate-sbom-prep.ps1
@@ -0,0 +1,21 @@
+Param(
+ [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
+)
+
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+Write-Host "Creating dir $ManifestDirPath"
+# create directory for sbom manifest to be placed
+if (!(Test-Path -path $ManifestDirPath))
+{
+ New-Item -ItemType Directory -path $ManifestDirPath
+ Write-Host "Successfully created directory $ManifestDirPath"
+}
+else{
+ Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
+}
+
+Write-Host "Updating artifact name"
+$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
+Write-Host "Artifact name $artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh
new file mode 100644
index 0000000000..d5c76dc827
--- /dev/null
+++ b/eng/common/generate-sbom-prep.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+. $scriptroot/pipeline-logging-functions.sh
+
+manifest_dir=$1
+
+if [ ! -d "$manifest_dir" ] ; then
+ mkdir -p "$manifest_dir"
+ echo "Sbom directory created." $manifest_dir
+else
+ Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
+fi
+
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+echo "Artifact name before : "$artifact_name
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
+echo "Artifact name after : "$safe_artifact_name
+export ARTIFACT_NAME=$safe_artifact_name
+echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
+
+exit 0
diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1
index db830c00a6..6c7a851a80 100644
--- a/eng/common/init-tools-native.ps1
+++ b/eng/common/init-tools-native.ps1
@@ -31,6 +31,10 @@ Wait time between retry attempts in seconds
.PARAMETER GlobalJsonFile
File path to global.json file
+.PARAMETER PathPromotion
+Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines)
+or break the build if a native tool is not found on the path (on a local dev machine)
+
.NOTES
#>
[CmdletBinding(PositionalBinding=$false)]
@@ -41,7 +45,8 @@ Param (
[switch] $Force = $False,
[int] $DownloadRetries = 5,
[int] $RetryWaitTimeInSeconds = 30,
- [string] $GlobalJsonFile
+ [string] $GlobalJsonFile,
+ [switch] $PathPromotion
)
if (!$GlobalJsonFile) {
@@ -77,53 +82,101 @@ try {
ConvertFrom-Json |
Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
if ($NativeTools) {
- $NativeTools.PSObject.Properties | ForEach-Object {
- $ToolName = $_.Name
- $ToolVersion = $_.Value
- $LocalInstallerArguments = @{ ToolName = "$ToolName" }
- $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
- $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
- $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
- $LocalInstallerArguments += @{ Version = "$ToolVersion" }
-
- if ($Verbose) {
- $LocalInstallerArguments += @{ Verbose = $True }
- }
- if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
- if($Force) {
- $LocalInstallerArguments += @{ Force = $True }
- }
- }
- if ($Clean) {
- $LocalInstallerArguments += @{ Clean = $True }
- }
-
- Write-Verbose "Installing $ToolName version $ToolVersion"
- Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
- & $InstallerPath @LocalInstallerArguments
- if ($LASTEXITCODE -Ne "0") {
- $errMsg = "$ToolName installation failed"
- if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
- $showNativeToolsWarning = $true
- if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
- $showNativeToolsWarning = $false
+ if ($PathPromotion -eq $True) {
+ if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+ $InstalledTools = @{}
+
+ if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
+ if ($ToolVersion -eq "latest") {
+ $ToolVersion = ""
+ }
+ $ArcadeToolsDirectory = "C:\arcade-tools"
+ if (-not (Test-Path $ArcadeToolsDirectory)) {
+ Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
+ exit 1
+ }
+ $ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0]
+ if ([string]::IsNullOrWhiteSpace($ToolDirectory)) {
+ Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
+ exit 1
}
- if ($showNativeToolsWarning) {
- Write-Warning $errMsg
+ $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt"
+ if (-not (Test-Path -Path "$BinPathFile")) {
+ Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool."
+ exit 1
}
- $toolInstallationFailure = $true
- } else {
- # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
- Write-Host $errMsg
- exit 1
+ $BinPath = Get-Content "$BinPathFile"
+ $ToolPath = Convert-Path -Path $BinPath
+ Write-Host "Adding $ToolName to the path ($ToolPath)..."
+ Write-Host "##vso[task.prependpath]$ToolPath"
+ $env:PATH = "$ToolPath;$env:PATH"
+ $InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
+ }
}
+ return $InstalledTools
+ } else {
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+
+ if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
+ Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
+ }
+ }
+ exit 0
+ }
+ } else {
+ $NativeTools.PSObject.Properties | ForEach-Object {
+ $ToolName = $_.Name
+ $ToolVersion = $_.Value
+ $LocalInstallerArguments = @{ ToolName = "$ToolName" }
+ $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+ $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+ $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+ $LocalInstallerArguments += @{ Version = "$ToolVersion" }
+
+ if ($Verbose) {
+ $LocalInstallerArguments += @{ Verbose = $True }
+ }
+ if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
+ if($Force) {
+ $LocalInstallerArguments += @{ Force = $True }
+ }
+ }
+ if ($Clean) {
+ $LocalInstallerArguments += @{ Clean = $True }
+ }
+
+ Write-Verbose "Installing $ToolName version $ToolVersion"
+ Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
+ & $InstallerPath @LocalInstallerArguments
+ if ($LASTEXITCODE -Ne "0") {
+ $errMsg = "$ToolName installation failed"
+ if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
+ $showNativeToolsWarning = $true
+ if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+ $showNativeToolsWarning = $false
+ }
+ if ($showNativeToolsWarning) {
+ Write-Warning $errMsg
+ }
+ $toolInstallationFailure = $true
+ } else {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host $errMsg
+ exit 1
+ }
+ }
+ }
+
+ if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+ # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+ Write-Host 'Native tools bootstrap failed'
+ exit 1
}
- }
-
- if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
- # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
- Write-Host 'Native tools bootstrap failed'
- exit 1
}
}
else {
@@ -139,7 +192,7 @@ try {
Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
return $InstallBin
}
- else {
+ elseif (-not ($PathPromotion)) {
Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
exit 1
}
@@ -149,4 +202,4 @@ catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
ExitWithExitCode 1
-}
+}
\ No newline at end of file
diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config
new file mode 100644
index 0000000000..19d3d311b1
--- /dev/null
+++ b/eng/common/internal/NuGet.config
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj
index beb9c4648e..7f5ce6d608 100644
--- a/eng/common/internal/Tools.csproj
+++ b/eng/common/internal/Tools.csproj
@@ -8,6 +8,9 @@
+
+
+
diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1
deleted file mode 100644
index de2d957922..0000000000
--- a/eng/common/post-build/add-build-to-channel.ps1
+++ /dev/null
@@ -1,48 +0,0 @@
-param(
- [Parameter(Mandatory=$true)][int] $BuildId,
- [Parameter(Mandatory=$true)][int] $ChannelId,
- [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
- [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
- [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
-)
-
-try {
- . $PSScriptRoot\post-build-utils.ps1
-
- # Check that the channel we are going to promote the build to exist
- $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
-
- if (!$channelInfo) {
- Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
- ExitWithExitCode 1
- }
-
- # Get info about which channel(s) the build has already been promoted to
- $buildInfo = Get-MaestroBuild -BuildId $BuildId
-
- if (!$buildInfo) {
- Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
- ExitWithExitCode 1
- }
-
- # Find whether the build is already assigned to the channel or not
- if ($buildInfo.channels) {
- foreach ($channel in $buildInfo.channels) {
- if ($channel.Id -eq $ChannelId) {
- Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
- ExitWithExitCode 0
- }
- }
- }
-
- Write-Host "Promoting build '$BuildId' to channel '$ChannelId'."
-
- Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
-
- Write-Host 'done.'
-}
-catch {
- Write-Host $_
- Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
- ExitWithExitCode 1
-}
diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1
index 63f3464c98..470d48455b 100644
--- a/eng/common/post-build/check-channel-consistency.ps1
+++ b/eng/common/post-build/check-channel-consistency.ps1
@@ -4,7 +4,15 @@ param(
)
try {
- . $PSScriptRoot\post-build-utils.ps1
+ $ErrorActionPreference = 'Stop'
+ Set-StrictMode -Version 2.0
+
+ # `tools.ps1` checks $ci to perform some actions. Since the post-build
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ $disableConfigureToolsetImport = $true
+ . $PSScriptRoot\..\tools.ps1
if ($PromoteToChannels -eq "") {
Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1
index dab3534ab5..9c81aa4391 100644
--- a/eng/common/post-build/nuget-validation.ps1
+++ b/eng/common/post-build/nuget-validation.ps1
@@ -2,20 +2,11 @@
# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage
param(
- [Parameter(Mandatory=$true)][string] $PackagesPath, # Path to where the packages to be validated are
- [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to
+ [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are
)
try {
- . $PSScriptRoot\post-build-utils.ps1
-
- $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
-
- New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
-
- Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
-
- & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
+ & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg
}
catch {
Write-Host $_.ScriptStackTrace
diff --git a/eng/common/post-build/nuget-verification.ps1 b/eng/common/post-build/nuget-verification.ps1
new file mode 100644
index 0000000000..a365194a93
--- /dev/null
+++ b/eng/common/post-build/nuget-verification.ps1
@@ -0,0 +1,121 @@
+<#
+.SYNOPSIS
+ Verifies that Microsoft NuGet packages have proper metadata.
+.DESCRIPTION
+ Downloads a verification tool and runs metadata validation on the provided NuGet packages. This script writes an
+ error if any of the provided packages fail validation. All arguments provided to this PowerShell script that do not
+ match PowerShell parameters are passed on to the verification tool downloaded during the execution of this script.
+.PARAMETER NuGetExePath
+ The path to the nuget.exe binary to use. If not provided, nuget.exe will be downloaded into the -DownloadPath
+ directory.
+.PARAMETER PackageSource
+ The package source to use to download the verification tool. If not provided, nuget.org will be used.
+.PARAMETER DownloadPath
+ The directory path to download the verification tool and nuget.exe to. If not provided,
+ %TEMP%\NuGet.VerifyNuGetPackage will be used.
+.PARAMETER args
+ Arguments that will be passed to the verification tool.
+.EXAMPLE
+ PS> .\verify.ps1 *.nupkg
+ Verifies the metadata of all .nupkg files in the currect working directory.
+.EXAMPLE
+ PS> .\verify.ps1 --help
+ Displays the help text of the downloaded verifiction tool.
+.LINK
+ https://github.com/NuGet/NuGetGallery/blob/master/src/VerifyMicrosoftPackage/README.md
+#>
+
+# This script was copied from https://github.com/NuGet/NuGetGallery/blob/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1
+
+[CmdletBinding(PositionalBinding = $false)]
+param(
+ [string]$NuGetExePath,
+ [string]$PackageSource = "https://api.nuget.org/v3/index.json",
+ [string]$DownloadPath,
+ [Parameter(ValueFromRemainingArguments = $true)]
+ [string[]]$args
+)
+
+# The URL to download nuget.exe.
+$nugetExeUrl = "https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe"
+
+# The package ID of the verification tool.
+$packageId = "NuGet.VerifyMicrosoftPackage"
+
+# The location that nuget.exe and the verification tool will be downloaded to.
+if (!$DownloadPath) {
+ $DownloadPath = (Join-Path $env:TEMP "NuGet.VerifyMicrosoftPackage")
+}
+
+$fence = New-Object -TypeName string -ArgumentList '=', 80
+
+# Create the download directory, if it doesn't already exist.
+if (!(Test-Path $DownloadPath)) {
+ New-Item -ItemType Directory $DownloadPath | Out-Null
+}
+Write-Host "Using download path: $DownloadPath"
+
+if ($NuGetExePath) {
+ $nuget = $NuGetExePath
+} else {
+ $downloadedNuGetExe = Join-Path $DownloadPath "nuget.exe"
+
+ # Download nuget.exe, if it doesn't already exist.
+ if (!(Test-Path $downloadedNuGetExe)) {
+ Write-Host "Downloading nuget.exe from $nugetExeUrl..."
+ $ProgressPreference = 'SilentlyContinue'
+ try {
+ Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe
+ $ProgressPreference = 'Continue'
+ } catch {
+ $ProgressPreference = 'Continue'
+ Write-Error $_
+ Write-Error "nuget.exe failed to download."
+ exit
+ }
+ }
+
+ $nuget = $downloadedNuGetExe
+}
+
+Write-Host "Using nuget.exe path: $nuget"
+Write-Host " "
+
+# Download the latest version of the verification tool.
+Write-Host "Downloading the latest version of $packageId from $packageSource..."
+Write-Host $fence
+& $nuget install $packageId `
+ -Prerelease `
+ -OutputDirectory $DownloadPath `
+ -Source $PackageSource
+Write-Host $fence
+Write-Host " "
+
+if ($LASTEXITCODE -ne 0) {
+ Write-Error "nuget.exe failed to fetch the verify tool."
+ exit
+}
+
+# Find the most recently downloaded tool
+Write-Host "Finding the most recently downloaded verification tool."
+$verifyProbePath = Join-Path $DownloadPath "$packageId.*"
+$verifyPath = Get-ChildItem -Path $verifyProbePath -Directory `
+ | Sort-Object -Property LastWriteTime -Descending `
+ | Select-Object -First 1
+$verify = Join-Path $verifyPath "tools\NuGet.VerifyMicrosoftPackage.exe"
+Write-Host "Using verification tool: $verify"
+Write-Host " "
+
+# Execute the verification tool.
+Write-Host "Executing the verify tool..."
+Write-Host $fence
+& $verify $args
+Write-Host $fence
+Write-Host " "
+
+# Respond to the exit code.
+if ($LASTEXITCODE -ne 0) {
+ Write-Error "The verify tool found some problems."
+} else {
+ Write-Output "The verify tool succeeded."
+}
diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1
deleted file mode 100644
index 534f6988d5..0000000000
--- a/eng/common/post-build/post-build-utils.ps1
+++ /dev/null
@@ -1,91 +0,0 @@
-# Most of the functions in this file require the variables `MaestroApiEndPoint`,
-# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
-
-$ErrorActionPreference = 'Stop'
-Set-StrictMode -Version 2.0
-
-# `tools.ps1` checks $ci to perform some actions. Since the post-build
-# scripts don't necessarily execute in the same agent that run the
-# build.ps1/sh script this variable isn't automatically set.
-$ci = $true
-$disableConfigureToolsetImport = $true
-. $PSScriptRoot\..\tools.ps1
-
-function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
- Validate-MaestroVars
-
- $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
- $headers.Add('Accept', $ContentType)
- $headers.Add('Authorization',"Bearer $MaestroApiAccessToken")
- return $headers
-}
-
-function Get-MaestroChannel([int]$ChannelId) {
- Validate-MaestroVars
-
- $apiHeaders = Create-MaestroApiRequestHeaders
- $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion"
-
- $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
- return $result
-}
-
-function Get-MaestroBuild([int]$BuildId) {
- Validate-MaestroVars
-
- $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
- $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion"
-
- $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
- return $result
-}
-
-function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
- Validate-MaestroVars
-
- $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository)
- $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
- $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion"
-
- $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
- return $result
-}
-
-function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
- Validate-MaestroVars
-
- $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
- $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
- Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
-}
-
-function Trigger-Subscription([string]$SubscriptionId) {
- Validate-MaestroVars
-
- $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
- $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
- Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
-}
-
-function Validate-MaestroVars {
- try {
- Get-Variable MaestroApiEndPoint | Out-Null
- Get-Variable MaestroApiVersion | Out-Null
- Get-Variable MaestroApiAccessToken | Out-Null
-
- if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
- Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
- ExitWithExitCode 1
- }
-
- if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
- Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
- ExitWithExitCode 1
- }
- }
- catch {
- Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
- Write-Host $_
- ExitWithExitCode 1
- }
-}
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index 2427ca6b6a..4ff587ca46 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -2,22 +2,21 @@ param(
[Parameter(Mandatory=$true)][int] $BuildId,
[Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
[Parameter(Mandatory=$true)][string] $AzdoToken,
- [Parameter(Mandatory=$true)][string] $MaestroToken,
- [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+ [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
[Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
- [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
- [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
- [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
- [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
- [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
- [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
+ [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
)
try {
- . $PSScriptRoot\post-build-utils.ps1
+ # `tools.ps1` checks $ci to perform some actions. Since the post-build
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ $disableConfigureToolsetImport = $true
+ . $PSScriptRoot\..\tools.ps1
- $darc = Get-Darc
+ $darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new()
@@ -35,35 +34,14 @@ try {
$optionalParams.Add("--no-wait") | Out-Null
}
- if ("false" -ne $PublishInstallersAndChecksums) {
- $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
- }
-
- if ("true" -eq $EnableNugetValidation) {
- $optionalParams.Add("--validate-nuget") | Out-Null
- }
-
- if ("true" -eq $EnableSourceLinkValidation) {
- $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null
- }
-
- if ("true" -eq $EnableSigningValidation) {
- $optionalParams.Add("--validate-signingchecksums") | Out-Null
-
- if ("" -ne $SigningValidationAdditionalParameters) {
- $optionalParams.Add("--signing-validation-parameters") | Out-Null
- $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null
- }
- }
-
& $darc add-build-to-channel `
- --id $buildId `
- --publishing-infra-version $PublishingInfraVersion `
- --default-channels `
- --source-branch main `
- --azdev-pat $AzdoToken `
- --bar-uri $MaestroApiEndPoint `
- --password $MaestroToken `
+ --id $buildId `
+ --publishing-infra-version $PublishingInfraVersion `
+ --default-channels `
+ --source-branch main `
+ --azdev-pat "$AzdoToken" `
+ --bar-uri "$MaestroApiEndPoint" `
+ --ci `
@optionalParams
if ($LastExitCode -ne 0) {
@@ -72,7 +50,7 @@ try {
}
Write-Host 'done.'
-}
+}
catch {
Write-Host $_
Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels."
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
index e8ab29afeb..5c4dd17dc3 100644
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -6,7 +6,15 @@ param(
[Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use
)
-. $PSScriptRoot\post-build-utils.ps1
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = $true
+. $PSScriptRoot\..\tools.ps1
# Cache/HashMap (File -> Exist flag) used to consult whether a file exist
# in the repository at a specific commit point. This is populated by inserting
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
index a5af041ba7..8857e3b840 100644
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -299,8 +299,6 @@ function InstallDotnetSymbol {
}
try {
- . $PSScriptRoot\post-build-utils.ps1
-
InstallDotnetSymbol
foreach ($Job in @(Get-Job)) {
diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1
deleted file mode 100644
index 55dea518ac..0000000000
--- a/eng/common/post-build/trigger-subscriptions.ps1
+++ /dev/null
@@ -1,64 +0,0 @@
-param(
- [Parameter(Mandatory=$true)][string] $SourceRepo,
- [Parameter(Mandatory=$true)][int] $ChannelId,
- [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
- [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
- [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
-)
-
-try {
- . $PSScriptRoot\post-build-utils.ps1
-
- # Get all the $SourceRepo subscriptions
- $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
- $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
-
- if (!$subscriptions) {
- Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
- ExitWithExitCode 0
- }
-
- $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
- $failedTriggeredSubscription = $false
-
- # Get all enabled subscriptions that need dependency flow on 'everyBuild'
- foreach ($subscription in $subscriptions) {
- if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
- Write-Host "Should trigger this subscription: ${$subscription.id}"
- [void]$subscriptionsToTrigger.Add($subscription.id)
- }
- }
-
- foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
- try {
- Write-Host "Triggering subscription '$subscriptionToTrigger'."
-
- Trigger-Subscription -SubscriptionId $subscriptionToTrigger
-
- Write-Host 'done.'
- }
- catch
- {
- Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
- Write-Host $_
- Write-Host $_.ScriptStackTrace
- $failedTriggeredSubscription = $true
- }
- }
-
- if ($subscriptionsToTrigger.Count -eq 0) {
- Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
- }
- elseif ($failedTriggeredSubscription) {
- Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
- ExitWithExitCode 1
- }
- else {
- Write-Host 'All subscriptions were triggered successfully!'
- }
-}
-catch {
- Write-Host $_.ScriptStackTrace
- Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
- ExitWithExitCode 1
-}
diff --git a/eng/common/retain-build.ps1 b/eng/common/retain-build.ps1
new file mode 100644
index 0000000000..e7ba975ade
--- /dev/null
+++ b/eng/common/retain-build.ps1
@@ -0,0 +1,45 @@
+
+Param(
+[Parameter(Mandatory=$true)][int] $buildId,
+[Parameter(Mandatory=$true)][string] $azdoOrgUri,
+[Parameter(Mandatory=$true)][string] $azdoProject,
+[Parameter(Mandatory=$true)][string] $token
+)
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+function Get-AzDOHeaders(
+ [string] $token)
+{
+ $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}"))
+ $headers = @{"Authorization"="Basic $base64AuthInfo"}
+ return $headers
+}
+
+function Update-BuildRetention(
+ [string] $azdoOrgUri,
+ [string] $azdoProject,
+ [int] $buildId,
+ [string] $token)
+{
+ $headers = Get-AzDOHeaders -token $token
+ $requestBody = "{
+ `"keepForever`": `"true`"
+ }"
+
+ $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0"
+ write-Host "Attempting to retain build using the following URI: ${requestUri} ..."
+
+ try {
+ Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json"
+ Write-Host "Updated retention settings for build ${buildId}."
+ }
+ catch {
+ Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription"
+ exit 1
+ }
+}
+
+Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token
+exit 0
diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
index 7ab9baac5c..b1bca63ab1 100644
--- a/eng/common/sdk-task.ps1
+++ b/eng/common/sdk-task.ps1
@@ -83,9 +83,6 @@ try {
}
if ($restore) {
- if ($ci) {
- Try-LogClientIpAddress
- }
Build 'Restore'
}
diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1
index 4999c30708..adea8e8da2 100644
--- a/eng/common/sdl/configure-sdl-tool.ps1
+++ b/eng/common/sdl/configure-sdl-tool.ps1
@@ -15,7 +15,9 @@ Param(
# Optional: Additional params to add to any tool using CredScan.
[string[]] $CrScanAdditionalRunConfigParams,
# Optional: Additional params to add to any tool using PoliCheck.
- [string[]] $PoliCheckAdditionalRunConfigParams
+ [string[]] $PoliCheckAdditionalRunConfigParams,
+ # Optional: Additional params to add to any tool using CodeQL/Semmle.
+ [string[]] $CodeQLAdditionalRunConfigParams
)
$ErrorActionPreference = 'Stop'
@@ -78,6 +80,11 @@ try {
$tool.Args += "Target < $TargetDirectory"
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
+ } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
+ if ($targetDirectory) {
+ $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+ }
+ $tool.Args += $CodeQLAdditionalRunConfigParams
}
# Create variable pointing to the args array directly so we can use splat syntax later.
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index 1157151f48..b9fe731796 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -34,6 +34,7 @@ Param(
[string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
[string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
[string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+ [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
[bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run
)
@@ -105,7 +106,8 @@ try {
-AzureDevOpsAccessToken $AzureDevOpsAccessToken `
-GuardianLoggerLevel $GuardianLoggerLevel `
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
- -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
+ -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
if ($BreakOnFailure) {
Exit-IfNZEC "Sdl"
}
diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1
new file mode 100644
index 0000000000..648c5068d7
--- /dev/null
+++ b/eng/common/sdl/sdl.ps1
@@ -0,0 +1,38 @@
+
+function Install-Gdn {
+ param(
+ [Parameter(Mandatory=$true)]
+ [string]$Path,
+
+ # If omitted, install the latest version of Guardian, otherwise install that specific version.
+ [string]$Version
+ )
+
+ $ErrorActionPreference = 'Stop'
+ Set-StrictMode -Version 2.0
+ $disableConfigureToolsetImport = $true
+ $global:LASTEXITCODE = 0
+
+ # `tools.ps1` checks $ci to perform some actions. Since the SDL
+ # scripts don't necessarily execute in the same agent that run the
+ # build.ps1/sh script this variable isn't automatically set.
+ $ci = $true
+ . $PSScriptRoot\..\tools.ps1
+
+ $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+
+ if ($Version) {
+ $argumentList += "-Version $Version"
+ }
+
+ Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
+
+ $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
+
+ if (!$gdnCliPath)
+ {
+ Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
+ }
+
+ return $gdnCliPath.FullName
+}
\ No newline at end of file
diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml
new file mode 100644
index 0000000000..21945ebdc9
--- /dev/null
+++ b/eng/common/templates-official/job/job.yml
@@ -0,0 +1,277 @@
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+parameters:
+# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ cancelTimeoutInMinutes: ''
+ condition: ''
+ container: ''
+ continueOnError: false
+ dependsOn: ''
+ displayName: ''
+ pool: ''
+ steps: []
+ strategy: ''
+ timeoutInMinutes: ''
+ variables: []
+ workspace: ''
+ templateContext: ''
+
+# Job base template specific parameters
+ # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
+ artifacts: ''
+ enableMicrobuild: false
+ enablePublishBuildArtifacts: false
+ enablePublishBuildAssets: false
+ enablePublishTestResults: false
+ enablePublishUsingPipelines: false
+ disableComponentGovernance: ''
+ mergeTestResults: false
+ testRunTitle: ''
+ testResultsFormat: ''
+ name: ''
+ preSteps: []
+ runAsPublic: false
+# Sbom related params
+ enableSbom: true
+ PackageVersion: 6.0.0
+ BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+
+jobs:
+- job: ${{ parameters.name }}
+
+ ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}:
+ cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }}
+
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+
+ ${{ if ne(parameters.container, '') }}:
+ container: ${{ parameters.container }}
+
+ ${{ if ne(parameters.continueOnError, '') }}:
+ continueOnError: ${{ parameters.continueOnError }}
+
+ ${{ if ne(parameters.dependsOn, '') }}:
+ dependsOn: ${{ parameters.dependsOn }}
+
+ ${{ if ne(parameters.displayName, '') }}:
+ displayName: ${{ parameters.displayName }}
+
+ ${{ if ne(parameters.pool, '') }}:
+ pool: ${{ parameters.pool }}
+
+ ${{ if ne(parameters.strategy, '') }}:
+ strategy: ${{ parameters.strategy }}
+
+ ${{ if ne(parameters.timeoutInMinutes, '') }}:
+ timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+
+ ${{ if ne(parameters.templateContext, '') }}:
+ templateContext: ${{ parameters.templateContext }}
+
+ variables:
+ - ${{ if ne(parameters.enableTelemetry, 'false') }}:
+ - name: DOTNET_CLI_TELEMETRY_PROFILE
+ value: '$(Build.Repository.Uri)'
+ - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
+ - name: EnableRichCodeNavigation
+ value: 'true'
+ # Retry signature validation up to three times, waiting 2 seconds between attempts.
+ # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
+ - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
+ value: 3,2000
+ - ${{ each variable in parameters.variables }}:
+ # handle name-value variable syntax
+ # example:
+ # - name: [key]
+ # value: [value]
+ - ${{ if ne(variable.name, '') }}:
+ - name: ${{ variable.name }}
+ value: ${{ variable.value }}
+
+ # handle variable groups
+ - ${{ if ne(variable.group, '') }}:
+ - group: ${{ variable.group }}
+
+ # handle key-value variable syntax.
+ # example:
+ # - [key]: [value]
+ - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}:
+ - ${{ each pair in variable }}:
+ - name: ${{ pair.key }}
+ value: ${{ pair.value }}
+
+ # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
+ - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - group: DotNet-HelixApi-Access
+
+ ${{ if ne(parameters.workspace, '') }}:
+ workspace: ${{ parameters.workspace }}
+
+ steps:
+ - ${{ if ne(parameters.preSteps, '') }}:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - task: MicroBuildSigningPlugin@4
+ displayName: Install MicroBuild plugin
+ inputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ env:
+ TeamName: $(_TeamName)
+ MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+
+ - task: NuGetAuthenticate@1
+
+ - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ buildType: current
+ artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+ targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+ itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
+ - ${{ each step in parameters.steps }}:
+ - ${{ step }}
+
+ - ${{ if eq(parameters.enableRichCodeNavigation, true) }}:
+ - task: RichCodeNavIndexer@0
+ displayName: RichCodeNav Upload
+ inputs:
+ languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
+ environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
+ richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
+ continueOnError: true
+
+ - template: /eng/common/templates-official/steps/component-governance.yml
+ parameters:
+ ${{ if eq(parameters.disableComponentGovernance, '') }}:
+ ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+ disableComponentGovernance: false
+ ${{ else }}:
+ disableComponentGovernance: true
+ ${{ else }}:
+ disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
+
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: MicroBuildCleanup@1
+ displayName: Execute Microbuild cleanup tasks
+ condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
+ env:
+ TeamName: $(_TeamName)
+
+ - ${{ if ne(parameters.artifacts.publish, '') }}:
+ - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+ - task: CopyFiles@2
+ displayName: Gather binaries for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/bin'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+ - task: CopyFiles@2
+ displayName: Gather packages for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/packages'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish pipeline artifacts
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+ PublishLocation: Container
+ ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+ - task: 1ES.PublishPipelineArtifact@1
+ inputs:
+ targetPath: 'artifacts/log'
+ artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+ displayName: 'Publish logs'
+ continueOnError: true
+ condition: always()
+
+ - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+ PublishLocation: Container
+ ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ continueOnError: true
+ condition: always()
+
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish XUnit Test Results
+ inputs:
+ testResultsFormat: 'xUnit'
+ testResultsFiles: '*.xml'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
+ - task: PublishTestResults@2
+ displayName: Publish TRX Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
+
+ - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+ - template: /eng/common/templates-official/steps/generate-sbom.yml
+ parameters:
+ PackageVersion: ${{ parameters.packageVersion}}
+ BuildDropPath: ${{ parameters.buildDropPath }}
diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml
new file mode 100644
index 0000000000..08df563759
--- /dev/null
+++ b/eng/common/templates-official/job/onelocbuild.yml
@@ -0,0 +1,109 @@
+parameters:
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool: ''
+
+ CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
+ GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
+
+ SourcesDirectory: $(Build.SourcesDirectory)
+ CreatePr: true
+ AutoCompletePr: false
+ ReusePr: true
+ UseLfLineEndings: true
+ UseCheckedInLocProjectJson: false
+ LanguageSet: VS_Main_Languages
+ LclSource: lclFilesInRepo
+ LclPackageId: ''
+ RepoType: gitHub
+ GitHubOrg: dotnet
+ MirrorRepo: ''
+ MirrorBranch: main
+ condition: ''
+
+jobs:
+- job: OneLocBuild
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: OneLocBuild
+
+ ${{ if ne(parameters.pool, '') }}:
+ pool: ${{ parameters.pool }}
+ ${{ if eq(parameters.pool, '') }}:
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+
+ variables:
+ - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+ - name: _GenerateLocProjectArguments
+ value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+ -LanguageSet "${{ parameters.LanguageSet }}"
+ -CreateNeutralXlfs
+ - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+ - name: _GenerateLocProjectArguments
+ value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+
+
+ steps:
+ - task: Powershell@2
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+ arguments: $(_GenerateLocProjectArguments)
+ displayName: Generate LocProject.json
+ condition: ${{ parameters.condition }}
+
+ - task: OneLocBuild@2
+ displayName: OneLocBuild
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ inputs:
+ locProj: eng/Localize/LocProject.json
+ outDir: $(Build.ArtifactStagingDirectory)
+ lclSource: ${{ parameters.LclSource }}
+ lclPackageId: ${{ parameters.LclPackageId }}
+ isCreatePrSelected: ${{ parameters.CreatePr }}
+ ${{ if eq(parameters.CreatePr, true) }}:
+ isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+ isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ isShouldReusePrSelected: ${{ parameters.ReusePr }}
+ packageSourceAuth: patAuth
+ patVariable: ${{ parameters.CeapexPat }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ repoType: ${{ parameters.RepoType }}
+ gitHubPatVariable: "${{ parameters.GithubPat }}"
+ ${{ if ne(parameters.MirrorRepo, '') }}:
+ isMirrorRepoSelected: true
+ gitHubOrganization: ${{ parameters.GitHubOrg }}
+ mirrorRepo: ${{ parameters.MirrorRepo }}
+ mirrorBranch: ${{ parameters.MirrorBranch }}
+ condition: ${{ parameters.condition }}
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish Localization Files
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish LocProject.json
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
+ PublishLocation: Container
+ ArtifactName: Loc
+ condition: ${{ parameters.condition }}
\ No newline at end of file
diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml
new file mode 100644
index 0000000000..e035ef719b
--- /dev/null
+++ b/eng/common/templates-official/job/publish-build-assets.yml
@@ -0,0 +1,128 @@
+parameters:
+ configuration: 'Debug'
+
+ # Optional: condition for the job to run
+ condition: ''
+
+ # Optional: 'true' if future jobs should run even if this job fails
+ continueOnError: false
+
+ # Optional: dependencies of the job
+ dependsOn: ''
+
+ # Optional: Include PublishBuildArtifacts task
+ enablePublishBuildArtifacts: false
+
+ # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+ pool: {}
+
+ # Optional: should run as a public build even in the internal project
+ # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ runAsPublic: false
+
+ # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+ publishUsingPipelines: false
+
+jobs:
+- job: Asset_Registry_Publish
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: Publish to Build Asset Registry
+
+ pool: ${{ parameters.pool }}
+
+ variables:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - name: _BuildConfig
+ value: ${{ parameters.configuration }}
+ - group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - name: runCodesignValidationInjection
+ value: false
+
+ steps:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download artifact
+ inputs:
+ artifactName: AssetManifests
+ downloadPath: '$(Build.StagingDirectory)/Download'
+ checkDownloadedFiles: true
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: NuGetAuthenticate@1
+
+ - task: PowerShell@2
+ displayName: Enable cross-org NuGet feed authentication
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
+
+ - task: AzureCLI@2
+ displayName: Publish Build Assets
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: eng\common\sdk-task.ps1
+ arguments: >
+ -task PublishBuildAssets
+ -restore
+ -msbuildEngine dotnet
+ /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+ /p:MaestroApiEndpoint=https://maestro.dot.net
+ /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
+ /p:Configuration=$(_BuildConfig)
+ /p:OfficialBuildId=$(Build.BuildNumber)
+
+ - task: powershell@2
+ displayName: Create ReleaseConfigs Artifact
+ inputs:
+ targetType: inline
+ script: |
+ New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force
+ $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt"
+ Add-Content -Path $filePath -Value $(BARBuildId)
+ Add-Content -Path $filePath -Value "$(DefaultChannels)"
+ Add-Content -Path $filePath -Value $(IsStableBuild)
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish ReleaseConfigs Artifact
+ inputs:
+ PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
+ PublishLocation: Container
+ ArtifactName: ReleaseConfigs
+
+ - task: powershell@2
+ displayName: Check if SymbolPublishingExclusionsFile.txt exists
+ inputs:
+ targetType: inline
+ script: |
+ $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
+ if(Test-Path -Path $symbolExclusionfile)
+ {
+ Write-Host "SymbolExclusionFile exists"
+ Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
+ }
+ else{
+ Write-Host "Symbols Exclusion file does not exists"
+ Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
+ }
+
+ - task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish SymbolPublishingExclusionsFile Artifact
+ condition: eq(variables['SymbolExclusionFile'], 'true')
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ PublishLocation: Container
+ ArtifactName: ReleaseConfigs
+
+ - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+ - template: /eng/common/templates-official/steps/publish-logs.yml
+ parameters:
+ JobLabel: 'Publish_Artifacts_Logs'
diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml
new file mode 100644
index 0000000000..a6f7efa776
--- /dev/null
+++ b/eng/common/templates-official/job/source-build.yml
@@ -0,0 +1,72 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. The template produces a server job with a
+ # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+ # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+ jobNamePrefix: 'Source_Build'
+
+ # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+ # managed-only repositories. This is an object with these properties:
+ #
+ # name: ''
+ # The name of the job. This is included in the job ID.
+ # targetRID: ''
+ # The name of the target RID to use, instead of the one auto-detected by Arcade.
+ # nonPortable: false
+ # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+ # linux-x64), and compiling against distro-provided packages rather than portable ones.
+ # skipPublishValidation: false
+ # Disables publishing validation. By default, a check is performed to ensure no packages are
+ # published by source-build.
+ # container: ''
+ # A container to use. Runs in docker.
+ # pool: {}
+ # A pool to use. Runs directly on an agent.
+ # buildScript: ''
+ # Specifies the build script to invoke to perform the build in the repo. The default
+ # './build.sh' should work for typical Arcade repositories, but this is customizable for
+ # difficult situations.
+ # jobProperties: {}
+ # A list of job properties to inject at the top level, for potential extensibility beyond
+ # container and pool.
+ platform: {}
+
+ # If set to true and running on a non-public project,
+ # Internal blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+ displayName: Source-Build (${{ parameters.platform.name }})
+
+ ${{ each property in parameters.platform.jobProperties }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ ${{ if ne(parameters.platform.container, '') }}:
+ container: ${{ parameters.platform.container }}
+
+ ${{ if eq(parameters.platform.pool, '') }}:
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ pool:
+ ${{ if eq(variables['System.TeamProject'], 'public') }}:
+ name: NetCore-Svc-Public
+ demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+ ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-mariner-2
+ os: linux
+ ${{ if ne(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.platform.pool }}
+
+ workspace:
+ clean: all
+
+ steps:
+ - ${{ if eq(parameters.enableInternalSources, true) }}:
+ - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
+ - template: /eng/common/templates-official/steps/source-build.yml
+ parameters:
+ platform: ${{ parameters.platform }}
diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml
new file mode 100644
index 0000000000..f6f0c15bfb
--- /dev/null
+++ b/eng/common/templates-official/job/source-index-stage1.yml
@@ -0,0 +1,74 @@
+parameters:
+ runAsPublic: false
+ sourceIndexUploadPackageVersion: 2.0.0-20240502.12
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
+ preSteps: []
+ binlogPath: artifacts/log/Debug/Build.binlog
+ pool:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+ condition: ''
+ dependsOn: ''
+
+jobs:
+- job: SourceIndexStage1
+ dependsOn: ${{ parameters.dependsOn }}
+ condition: ${{ parameters.condition }}
+ variables:
+ - name: SourceIndexUploadPackageVersion
+ value: ${{ parameters.sourceIndexUploadPackageVersion }}
+ - name: SourceIndexProcessBinlogPackageVersion
+ value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
+ - name: SourceIndexPackageSource
+ value: ${{ parameters.sourceIndexPackageSource }}
+ - name: BinlogPath
+ value: ${{ parameters.binlogPath }}
+
+ pool: ${{ parameters.pool }}
+ steps:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
+
+ - task: UseDotNet@2
+ displayName: Use .NET 8 SDK
+ inputs:
+ packageType: sdk
+ version: 8.0.x
+ installationPath: $(Agent.TempDirectory)/dotnet
+ workingDirectory: $(Agent.TempDirectory)
+
+ - script: |
+ $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ displayName: Download Tools
+ # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+ workingDirectory: $(Agent.TempDirectory)
+
+ - script: ${{ parameters.sourceIndexBuildCommand }}
+ displayName: Build Repository
+
+ - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: Process Binlog into indexable sln
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: AzureCLI@2
+ displayName: Get stage 1 auth token
+ inputs:
+ azureSubscription: 'SourceDotNet Stage1 Publish'
+ addSpnToEnvironment: true
+ scriptType: 'ps'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+ echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
+
+ - script: |
+ az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
+ displayName: "Login to Azure"
+
+ - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
+ displayName: Upload stage1 artifacts to source index
diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml
new file mode 100644
index 0000000000..0bf7ee29f4
--- /dev/null
+++ b/eng/common/templates-official/jobs/codeql-build.yml
@@ -0,0 +1,31 @@
+parameters:
+ # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+ continueOnError: false
+ # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ jobs: []
+ # Optional: if specified, restore and use this version of Guardian instead of the default.
+ overrideGuardianVersion: ''
+
+jobs:
+- template: /eng/common/templates-official/jobs/jobs.yml
+ parameters:
+ enableMicrobuild: false
+ enablePublishBuildArtifacts: false
+ enablePublishTestResults: false
+ enablePublishBuildAssets: false
+ enablePublishUsingPipelines: false
+ enableTelemetry: true
+
+ variables:
+ - group: Publish-Build-Assets
+ # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+ # sync with the packages.config file.
+ - name: DefaultGuardianVersion
+ value: 0.110.1
+ - name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+ - name: GuardianVersion
+ value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+
+ jobs: ${{ parameters.jobs }}
+
diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml
new file mode 100644
index 0000000000..c124aa9957
--- /dev/null
+++ b/eng/common/templates-official/jobs/jobs.yml
@@ -0,0 +1,100 @@
+parameters:
+ # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+ continueOnError: false
+
+ # Optional: Include PublishBuildArtifacts task
+ enablePublishBuildArtifacts: false
+
+ # Optional: Enable publishing using release pipelines
+ enablePublishUsingPipelines: false
+
+ # Optional: Enable running the source-build jobs to build repo from source
+ enableSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates-official/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
+ graphFileGeneration:
+ # Optional: Enable generating the graph files at the end of the build
+ enabled: false
+ # Optional: Include toolset dependencies in the generated graph files
+ includeToolset: false
+
+ # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ jobs: []
+
+ # Optional: Override automatically derived dependsOn value for "publish build assets" job
+ publishBuildAssetsDependsOn: ''
+
+ # Optional: should run as a public build even in the internal project
+ # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ runAsPublic: false
+
+ enableSourceIndex: false
+ sourceIndexParams: {}
+
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
+jobs:
+- ${{ each job in parameters.jobs }}:
+ - template: ../job/job.yml
+ parameters:
+ # pass along parameters
+ ${{ each parameter in parameters }}:
+ ${{ if ne(parameter.key, 'jobs') }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+ # pass along job properties
+ ${{ each property in job }}:
+ ${{ if ne(property.key, 'job') }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ name: ${{ job.job }}
+
+- ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - template: /eng/common/templates-official/jobs/source-build.yml
+ parameters:
+ allCompletedJobId: Source_Build_Complete
+ ${{ each parameter in parameters.sourceBuildParameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
+ - template: ../job/source-index-stage1.yml
+ parameters:
+ runAsPublic: ${{ parameters.runAsPublic }}
+ ${{ each parameter in parameters.sourceIndexParams }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
+ - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - template: ../job/publish-build-assets.yml
+ parameters:
+ continueOnError: ${{ parameters.continueOnError }}
+ dependsOn:
+ - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+ - ${{ job.job }}
+ - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.jobs }}:
+ - ${{ job.job }}
+ - ${{ if eq(parameters.enableSourceBuild, true) }}:
+ - Source_Build_Complete
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ else }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+
+ runAsPublic: ${{ parameters.runAsPublic }}
+ publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+ enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml
new file mode 100644
index 0000000000..9f60019cb0
--- /dev/null
+++ b/eng/common/templates-official/jobs/source-build.yml
@@ -0,0 +1,54 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+ # well as an optional server job that completes when all platform jobs complete.
+
+ # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+ # not included. Existing repo pipelines can use this job depend on all source-build jobs
+ # completing without maintaining a separate list of every single job ID: just depend on this one
+ # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+ allCompletedJobId: ''
+
+ # See /eng/common/templates-official/job/source-build.yml
+ jobNamePrefix: 'Source_Build'
+
+ # This is the default platform provided by Arcade, intended for use by a managed-only repo.
+ defaultManagedPlatform:
+ name: 'Managed'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7'
+
+ # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+ # object in this array is sent to the job template as 'platform'. If no platforms are specified,
+ # one job runs on 'defaultManagedPlatform'.
+ platforms: []
+
+ # If set to true and running on a non-public project,
+ # Internal nuget and blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+ - job: ${{ parameters.allCompletedJobId }}
+ displayName: Source-Build Complete
+ pool: server
+ dependsOn:
+ - ${{ each platform in parameters.platforms }}:
+ - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+ - ${{ if eq(length(parameters.platforms), 0) }}:
+ - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+ - template: /eng/common/templates-official/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ platform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
+
+- ${{ if eq(length(parameters.platforms), 0) }}:
+ - template: /eng/common/templates-official/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ parameters.defaultManagedPlatform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml
new file mode 100644
index 0000000000..7e1af5af50
--- /dev/null
+++ b/eng/common/templates-official/post-build/common-variables.yml
@@ -0,0 +1,23 @@
+variables:
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - group: DotNet-Blob-Feed
+ - group: DotNet-DotNetCli-Storage
+ - group: Publish-Build-Assets
+
+ # Whether the build is internal or not
+ - name: IsInternalBuild
+ value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
+
+ # Default Maestro++ API Endpoint and API Version
+ - name: MaestroApiEndPoint
+ value: "https://maestro.dot.net"
+ - name: MaestroApiVersion
+ value: "2020-02-20"
+
+ - name: SourceLinkCLIVersion
+ value: 3.0.0
+ - name: SymbolToolVersion
+ value: 1.0.1
+
+ - name: runCodesignValidationInjection
+ value: false
diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml
new file mode 100644
index 0000000000..7a83f6c4e3
--- /dev/null
+++ b/eng/common/templates-official/post-build/post-build.yml
@@ -0,0 +1,280 @@
+parameters:
+ # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+ # Publishing V1 is no longer supported
+ # Publishing V2 is no longer supported
+ # Publishing V3 is the default
+ - name: publishingInfraVersion
+ displayName: Which version of publishing should be used to promote the build definition?
+ type: number
+ default: 3
+ values:
+ - 3
+
+ - name: BARBuildId
+ displayName: BAR Build Id
+ type: number
+ default: 0
+
+ - name: PromoteToChannelIds
+ displayName: Channel to promote BARBuildId to
+ type: string
+ default: ''
+
+ - name: enableSourceLinkValidation
+ displayName: Enable SourceLink validation
+ type: boolean
+ default: false
+
+ - name: enableSigningValidation
+ displayName: Enable signing validation
+ type: boolean
+ default: true
+
+ - name: enableSymbolValidation
+ displayName: Enable symbol validation
+ type: boolean
+ default: false
+
+ - name: enableNugetValidation
+ displayName: Enable NuGet validation
+ type: boolean
+ default: true
+
+ - name: publishInstallersAndChecksums
+ displayName: Publish installers and checksums
+ type: boolean
+ default: true
+
+ - name: SDLValidationParameters
+ type: object
+ default:
+ enable: false
+ continueOnError: false
+ params: ''
+ artifactNames: ''
+ downloadArtifacts: true
+
+ # These parameters let the user customize the call to sdk-task.ps1 for publishing
+ # symbols & general artifacts as well as for signing validation
+ - name: symbolPublishingAdditionalParameters
+ displayName: Symbol publishing additional parameters
+ type: string
+ default: ''
+
+ - name: artifactsPublishingAdditionalParameters
+ displayName: Artifact publishing additional parameters
+ type: string
+ default: ''
+
+ - name: signingValidationAdditionalParameters
+ displayName: Signing validation additional parameters
+ type: string
+ default: ''
+
+ # Which stages should finish execution before post-build stages start
+ - name: validateDependsOn
+ type: object
+ default:
+ - build
+
+ - name: publishDependsOn
+ type: object
+ default:
+ - Validate
+
+stages:
+- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ - stage: Validate
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Validate Build Assets
+ variables:
+ - template: common-variables.yml
+ jobs:
+ - job:
+ displayName: NuGet Validation
+ condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+
+ steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
+ arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
+
+ - job:
+ displayName: Signing Validation
+ condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+ steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: PackageArtifacts
+ checkDownloadedFiles: true
+ itemPattern: |
+ **
+ !**/Microsoft.SourceBuild.Intermediate.*.nupkg
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@1
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ # Signing validation will optionally work with the buildmanifest file which is downloaded from
+ # Azure DevOps above.
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task SigningValidation -restore -msbuildEngine vs
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
+ /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+ ${{ parameters.signingValidationAdditionalParameters }}
+
+ - template: ../steps/publish-logs.yml
+ parameters:
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
+
+ - job:
+ displayName: SourceLink Validation
+ condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+ steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: specific
+ buildVersionToDownload: specific
+ project: $(AzDOProjectName)
+ pipeline: $(AzDOPipelineId)
+ buildId: $(AzDOBuildId)
+ artifactName: BlobArtifacts
+ checkDownloadedFiles: true
+
+ - task: PowerShell@2
+ displayName: Validate
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+ -ExtractPath $(Agent.BuildDirectory)/Extract/
+ -GHRepoName $(Build.Repository.Name)
+ -GHCommit $(Build.SourceVersion)
+ -SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
+
+- stage: publish_using_darc
+ ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.publishDependsOn }}
+ ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Publish using Darc
+ variables:
+ - template: common-variables.yml
+ jobs:
+ - job:
+ displayName: Publish Using Darc
+ timeoutInMinutes: 120
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: AzurePipelines-EO
+ image: 1ESPT-Windows2022
+ demands: Cmd
+ os: windows
+ # If it's not devdiv, it's dnceng
+ ${{ else }}:
+ name: NetCore1ESPool-Svc-Internal
+ image: 1es-windows-2022
+ os: windows
+ steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - task: AzureCLI@2
+ displayName: Publish Using Darc
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: -BuildId $(BARBuildId)
+ -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+ -AzdoToken '$(System.AccessToken)'
+ -WaitPublishingFinish true
+ -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+ -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml
new file mode 100644
index 0000000000..af3a6eb12f
--- /dev/null
+++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml
@@ -0,0 +1,70 @@
+parameters:
+ BARBuildId: ''
+ PromoteToChannelIds: ''
+
+steps:
+ - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Release Configs
+ inputs:
+ buildType: current
+ artifactName: ReleaseConfigs
+ checkDownloadedFiles: true
+
+ - task: AzureCLI@2
+ name: setReleaseVars
+ displayName: Set Release Configs Vars
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: pscore
+ scriptLocation: inlineScript
+ inlineScript: |
+ try {
+ if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
+ $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+ $BarId = $Content | Select -Index 0
+ $Channels = $Content | Select -Index 1
+ $IsStableBuild = $Content | Select -Index 2
+
+ $AzureDevOpsProject = $Env:System_TeamProject
+ $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
+ $AzureDevOpsBuildId = $Env:Build_BuildId
+ }
+ else {
+ . $(Build.SourcesDirectory)\eng\common\tools.ps1
+ $darc = Get-Darc
+ $buildInfo = & $darc get-build `
+ --id ${{ parameters.BARBuildId }} `
+ --extended `
+ --output-format json `
+ --ci `
+ | convertFrom-Json
+
+ $BarId = ${{ parameters.BARBuildId }}
+ $Channels = $Env:PromoteToMaestroChannels -split ","
+ $Channels = $Channels -join "]["
+ $Channels = "[$Channels]"
+
+ $IsStableBuild = $buildInfo.stable
+ $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+ $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+ $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+ }
+
+ Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId"
+ Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels"
+ Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild"
+
+ Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject"
+ Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId"
+ Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId"
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+ }
+ env:
+ PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml
new file mode 100644
index 0000000000..12527b80ea
--- /dev/null
+++ b/eng/common/templates-official/steps/component-governance.yml
@@ -0,0 +1,10 @@
+parameters:
+ disableComponentGovernance: false
+
+steps:
+- ${{ if eq(parameters.disableComponentGovernance, 'true') }}:
+ - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
+ displayName: Set skipComponentGovernanceDetection variable
+- ${{ if ne(parameters.disableComponentGovernance, 'true') }}:
+ - task: ComponentGovernanceComponentDetection@0
+ continueOnError: true
\ No newline at end of file
diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml
new file mode 100644
index 0000000000..93a8394a66
--- /dev/null
+++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml
@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+ type: string
+ default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+ type: string
+ default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+ - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+ parameters:
+ federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+ outputVariableName: ${{ parameters.outputVariableName }}
+ expiryInHours: ${{ parameters.expiryInHours }}
+ base64Encode: ${{ parameters.base64Encode }}
+ storageAccount: dotnetbuilds
+ container: internal
+ permissions: rl
diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml
new file mode 100644
index 0000000000..9b4a5ffa30
--- /dev/null
+++ b/eng/common/templates-official/steps/execute-codeql.yml
@@ -0,0 +1,32 @@
+parameters:
+ # Language that should be analyzed. Defaults to csharp
+ language: csharp
+ # Build Commands
+ buildCommands: ''
+ overrideParameters: '' # Optional: to override values for parameters.
+ additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+ # Optional: if specified, restore and use this version of Guardian instead of the default.
+ overrideGuardianVersion: ''
+ # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+ # diagnosis of problems with specific tool configurations.
+ publishGuardianDirectoryToPipeline: false
+ # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+ # parameters rather than relying on YAML. It may be better to use a local script, because you can
+ # reproduce results locally without piecing together a command based on the YAML.
+ executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
+ # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+ # 'continueOnError', the parameter value is not correctly picked up.
+ # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+ # optional: determines whether to continue the build if the step errors;
+ sdlContinueOnError: false
+
+steps:
+- template: /eng/common/templates-official/steps/execute-sdl.yml
+ parameters:
+ overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
+ executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
+ overrideParameters: ${{ parameters.overrideParameters }}
+ additionalParameters: '${{ parameters.additionalParameters }}
+ -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")'
+ publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
+ sdlContinueOnError: ${{ parameters.sdlContinueOnError }}
\ No newline at end of file
diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml
new file mode 100644
index 0000000000..7fc4f35802
--- /dev/null
+++ b/eng/common/templates-official/steps/generate-sbom.yml
@@ -0,0 +1,44 @@
+# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
+# PackageName - The name of the package this SBOM represents.
+# PackageVersion - The version of the package this SBOM represents.
+# ManifestDirPath - The path of the directory where the generated manifest files will be placed
+
+parameters:
+ PackageVersion: 6.0.0
+ BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+ PackageName: '.NET'
+ ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
+ sbomContinueOnError: true
+
+steps:
+- task: PowerShell@2
+ displayName: Prep for SBOM generation in (Non-linux)
+ condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
+ inputs:
+ filePath: ./eng/common/generate-sbom-prep.ps1
+ arguments: ${{parameters.manifestDirPath}}
+
+# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
+- script: |
+ chmod +x ./eng/common/generate-sbom-prep.sh
+ ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
+ displayName: Prep for SBOM generation in (Linux)
+ condition: eq(variables['Agent.Os'], 'Linux')
+ continueOnError: ${{ parameters.sbomContinueOnError }}
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+ displayName: 'Generate SBOM manifest'
+ continueOnError: ${{ parameters.sbomContinueOnError }}
+ inputs:
+ PackageName: ${{ parameters.packageName }}
+ BuildDropPath: ${{ parameters.buildDropPath }}
+ PackageVersion: ${{ parameters.packageVersion }}
+ ManifestDirPath: ${{ parameters.manifestDirPath }}
+
+- task: 1ES.PublishPipelineArtifact@1
+ displayName: Publish SBOM manifest
+ continueOnError: ${{parameters.sbomContinueOnError}}
+ inputs:
+ targetPath: '${{parameters.manifestDirPath}}'
+ artifactName: $(ARTIFACT_NAME)
+
diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml
new file mode 100644
index 0000000000..c0e8f91317
--- /dev/null
+++ b/eng/common/templates-official/steps/get-delegation-sas.yml
@@ -0,0 +1,43 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: false
+- name: storageAccount
+ type: string
+- name: container
+ type: string
+- name: permissions
+ type: string
+ default: 'rl'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ # Calculate the expiration of the SAS token and convert to UTC
+ $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+
+ if ('${{ parameters.base64Encode }}' -eq 'true') {
+ $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+ }
+
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml
new file mode 100644
index 0000000000..e3786cef6d
--- /dev/null
+++ b/eng/common/templates-official/steps/get-federated-access-token.yml
@@ -0,0 +1,28 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+ type: string
+ default: '499b84ac-1321-427f-aa17-267ca6975798'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Getting federated access token for feeds'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+ exit 1
+ }
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken"
diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml
new file mode 100644
index 0000000000..04012fed18
--- /dev/null
+++ b/eng/common/templates-official/steps/publish-logs.yml
@@ -0,0 +1,23 @@
+parameters:
+ StageLabel: ''
+ JobLabel: ''
+
+steps:
+- task: Powershell@2
+ displayName: Prepare Binlogs to Upload
+ inputs:
+ targetType: inline
+ script: |
+ New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ continueOnError: true
+ condition: always()
+
+- task: 1ES.PublishBuildArtifacts@1
+ displayName: Publish Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+ PublishLocation: Container
+ ArtifactName: PostBuildLogs
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml
new file mode 100644
index 0000000000..83d97a26a0
--- /dev/null
+++ b/eng/common/templates-official/steps/retain-build.yml
@@ -0,0 +1,28 @@
+parameters:
+ # Optional azure devops PAT with build execute permissions for the build's organization,
+ # only needed if the build that should be retained ran on a different organization than
+ # the pipeline where this template is executing from
+ Token: ''
+ # Optional BuildId to retain, defaults to the current running build
+ BuildId: ''
+ # Azure devops Organization URI for the build in the https://dev.azure.com/ format.
+ # Defaults to the organization the current pipeline is running on
+ AzdoOrgUri: '$(System.CollectionUri)'
+ # Azure devops project for the build. Defaults to the project the current pipeline is running on
+ AzdoProject: '$(System.TeamProject)'
+
+steps:
+ - task: powershell@2
+ inputs:
+ targetType: 'filePath'
+ filePath: eng/common/retain-build.ps1
+ pwsh: true
+ arguments: >
+ -AzdoOrgUri: ${{parameters.AzdoOrgUri}}
+ -AzdoProject ${{parameters.AzdoProject}}
+ -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }}
+ -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}}
+ displayName: Enable permanent build retention
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ BUILD_ID: $(Build.BuildId)
\ No newline at end of file
diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml
new file mode 100644
index 0000000000..cd02ae1607
--- /dev/null
+++ b/eng/common/templates-official/steps/send-to-helix.yml
@@ -0,0 +1,94 @@
+# Please remember to update the documentation if you make changes to these parameters!
+parameters:
+ HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
+ HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+ HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+ HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
+ HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+ HelixConfiguration: '' # optional -- additional property attached to a job
+ HelixPreCommands: '' # optional -- commands to run before Helix work item execution
+ HelixPostCommands: '' # optional -- commands to run after Helix work item execution
+ WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+ WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+ WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+ CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
+ XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+ XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects
+ XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects
+ XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
+ XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+ IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+ DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+ DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
+ EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
+ WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+ IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+ HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
+ Creator: '' # optional -- if the build is external, use this to specify who is sending the job
+ DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
+ condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
+ continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
+
+steps:
+ - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+ displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
+ env:
+ BuildConfig: $(_BuildConfig)
+ HelixSource: ${{ parameters.HelixSource }}
+ HelixType: ${{ parameters.HelixType }}
+ HelixBuild: ${{ parameters.HelixBuild }}
+ HelixConfiguration: ${{ parameters.HelixConfiguration }}
+ HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+ HelixAccessToken: ${{ parameters.HelixAccessToken }}
+ HelixPreCommands: ${{ parameters.HelixPreCommands }}
+ HelixPostCommands: ${{ parameters.HelixPostCommands }}
+ WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+ WorkItemCommand: ${{ parameters.WorkItemCommand }}
+ WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+ CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+ XUnitProjects: ${{ parameters.XUnitProjects }}
+ XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+ XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+ XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+ XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+ IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+ DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+ DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+ EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+ WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
+ Creator: ${{ parameters.Creator }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
+ - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+ displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
+ env:
+ BuildConfig: $(_BuildConfig)
+ HelixSource: ${{ parameters.HelixSource }}
+ HelixType: ${{ parameters.HelixType }}
+ HelixBuild: ${{ parameters.HelixBuild }}
+ HelixConfiguration: ${{ parameters.HelixConfiguration }}
+ HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+ HelixAccessToken: ${{ parameters.HelixAccessToken }}
+ HelixPreCommands: ${{ parameters.HelixPreCommands }}
+ HelixPostCommands: ${{ parameters.HelixPostCommands }}
+ WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+ WorkItemCommand: ${{ parameters.WorkItemCommand }}
+ WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
+ CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+ XUnitProjects: ${{ parameters.XUnitProjects }}
+ XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }}
+ XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+ XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+ XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+ IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+ DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+ DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+ EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+ WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
+ Creator: ${{ parameters.Creator }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
+ continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml
new file mode 100644
index 0000000000..9eb7e51456
--- /dev/null
+++ b/eng/common/templates-official/steps/source-build.yml
@@ -0,0 +1,108 @@
+parameters:
+ # This template adds arcade-powered source-build to CI.
+
+ # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+ # infra has a careful build methodology that must be followed. For example, a repo
+ # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+ # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+ # GitHub. Using this steps template leaves room for that infra to be included.
+
+ # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml'
+ # for details. The entire object is described in the 'job' template for simplicity, even though
+ # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+ platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+ set -x
+ df -h
+
+ # If building on the internal project, the artifact feeds variable may be available (usually only if needed)
+ # In that case, call the feed setup script to add internal feeds corresponding to public ones.
+ # In addition, add an msbuild argument to copy the WIP from the repo to the target build location.
+ # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those
+ # changes.
+ $internalRestoreArgs=
+ if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then
+ # Temporarily work around https://github.com/dotnet/arcade/issues/7709
+ chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+ $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw)
+ internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
+
+ # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
+ # This only works if there is a username/email configured, which won't be the case in most CI runs.
+ git config --get user.email
+ if [ $? -ne 0 ]; then
+ git config user.email dn-bot@microsoft.com
+ git config user.name dn-bot
+ fi
+ fi
+
+ # If building on the internal project, the internal storage variable may be available (usually only if needed)
+ # In that case, add variables to allow the download of internal runtimes if the specified versions are not found
+ # in the default public locations.
+ internalRuntimeDownloadArgs=
+ if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
+ internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
+ fi
+
+ buildConfig=Release
+ # Check if AzDO substitutes in a build config from a variable, and use it if so.
+ if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+ buildConfig='$(_BuildConfig)'
+ fi
+
+ officialBuildArgs=
+ if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+ officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+ fi
+
+ targetRidArgs=
+ if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+ targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+ fi
+
+ runtimeOsArgs=
+ if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
+ runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
+ fi
+
+ publishArgs=
+ if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
+ publishArgs='--publish'
+ fi
+
+ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+ --configuration $buildConfig \
+ --restore --build --pack $publishArgs -bl \
+ $officialBuildArgs \
+ $internalRuntimeDownloadArgs \
+ $internalRestoreArgs \
+ $targetRidArgs \
+ $runtimeOsArgs \
+ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+ /p:ArcadeBuildFromSource=true
+ displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+ displayName: Prepare BuildLogs staging directory
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)'
+ Contents: |
+ **/*.log
+ **/*.binlog
+ artifacts/source-build/self/prebuilt-report/**
+ TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+ CleanTargetFolder: true
+ continueOnError: true
+ condition: succeededOrFailed()
+
+- task: 1ES.PublishPipelineArtifact@1
+ displayName: Publish BuildLogs
+ inputs:
+ targetPath: '$(Build.StagingDirectory)/BuildLogs'
+ artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+ continueOnError: true
+ condition: succeededOrFailed()
diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml
new file mode 100644
index 0000000000..1a860bd040
--- /dev/null
+++ b/eng/common/templates-official/variables/sdl-variables.yml
@@ -0,0 +1,7 @@
+variables:
+# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+# sync with the packages.config file.
+- name: DefaultGuardianVersion
+ value: 0.110.1
+- name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
\ No newline at end of file
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index 69eb67849d..aaeb83b4dc 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -29,14 +29,6 @@ parameters:
# Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
# not pipeline artifacts, so doesn't affect the use of this parameter.
pipelineArtifactNames: []
- # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
- # downloaded from. By default, uses runtime expressions to decide based on the variables set by
- # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
- # running without Maestro++/BAR involved, or to download artifacts from a specific existing build
- # to iterate quickly on SDL changes.
- AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
jobs:
- job: Run_SDL
@@ -51,24 +43,24 @@ jobs:
value: ${{ parameters.AzDOPipelineId }}
- name: AzDOBuildId
value: ${{ parameters.AzDOBuildId }}
- # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
- # sync with the packages.config file.
- - name: DefaultGuardianVersion
- value: 0.53.3
+ - template: /eng/common/templates/variables/sdl-variables.yml
- name: GuardianVersion
value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
- - name: GuardianPackagesConfigFile
- value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
pool:
- # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
- ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
- name: Hosted VS2017
- ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
- vmImage: windows-2019
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
steps:
- checkout: self
clean: true
+ - template: /eng/common/templates/post-build/setup-maestro-vars.yml
+
- ${{ if ne(parameters.downloadArtifacts, 'false')}}:
- ${{ if ne(parameters.artifactNames, '') }}:
- ${{ each artifactName in parameters.artifactNames }}:
@@ -129,57 +121,11 @@ jobs:
displayName: Extract Archive Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
- - ${{ if ne(parameters.overrideGuardianVersion, '') }}:
- - powershell: |
- $content = Get-Content $(GuardianPackagesConfigFile)
-
- Write-Host "packages.config content was:`n$content"
-
- $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
- $content | Set-Content $(GuardianPackagesConfigFile)
-
- Write-Host "packages.config content updated to:`n$content"
- displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
-
- - task: NuGetToolInstaller@1
- displayName: 'Install NuGet.exe'
- - task: NuGetCommand@2
- displayName: 'Install Guardian'
- inputs:
- restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
- feedsToUse: config
- nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
- externalFeedCredentials: GuardianConnect
- restoreDirectory: $(Build.SourcesDirectory)\.packages
-
- - ${{ if ne(parameters.overrideParameters, '') }}:
- - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
- displayName: Execute SDL
- continueOnError: ${{ parameters.sdlContinueOnError }}
- - ${{ if eq(parameters.overrideParameters, '') }}:
- - powershell: ${{ parameters.executeAllSdlToolsScript }}
- -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion)
- -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
- -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
- ${{ parameters.additionalParameters }}
- displayName: Execute SDL
- continueOnError: ${{ parameters.sdlContinueOnError }}
-
- - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
- # We want to publish the Guardian results and configuration for easy diagnosis. However, the
- # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
- # tooling files. Some of these files are large and aren't useful during an investigation, so
- # exclude them by simply deleting them before publishing. (As of writing, there is no documented
- # way to selectively exclude a dir from the pipeline artifact publish task.)
- - task: DeleteFiles@1
- displayName: Delete Guardian dependencies to avoid uploading
- inputs:
- SourceFolder: $(Agent.BuildDirectory)/.gdn
- Contents: |
- c
- i
- condition: succeededOrFailed()
- - publish: $(Agent.BuildDirectory)/.gdn
- artifact: GuardianConfiguration
- displayName: Publish GuardianConfiguration
- condition: succeededOrFailed()
+ - template: /eng/common/templates/steps/execute-sdl.yml
+ parameters:
+ overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
+ executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
+ overrideParameters: ${{ parameters.overrideParameters }}
+ additionalParameters: ${{ parameters.additionalParameters }}
+ publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
+ sdlContinueOnError: ${{ parameters.sdlContinueOnError }}
diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml
deleted file mode 100644
index e54ce956f9..0000000000
--- a/eng/common/templates/job/generate-graph-files.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-parameters:
- # Optional: dependencies of the job
- dependsOn: ''
-
- # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
- pool: {}
-
- # Optional: Include toolset dependencies in the generated graph files
- includeToolset: false
-
-jobs:
-- job: Generate_Graph_Files
-
- dependsOn: ${{ parameters.dependsOn }}
-
- displayName: Generate Graph Files
-
- pool: ${{ parameters.pool }}
-
- variables:
- # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
- # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat
- - group: Publish-Build-Assets
- - group: DotNet-AllOrgs-Darc-Pats
- - name: _GraphArguments
- value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT)
- -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat)
- -barToken $(MaestroAccessToken)
- -outputFolder '$(Build.StagingDirectory)/GraphFiles/'
- - ${{ if ne(parameters.includeToolset, 'false') }}:
- - name: _GraphArguments
- value: ${{ variables._GraphArguments }} -includeToolset
-
- steps:
- - task: PowerShell@2
- displayName: Generate Graph Files
- inputs:
- filePath: eng\common\generate-graph-files.ps1
- arguments: $(_GraphArguments)
- continueOnError: true
- - task: PublishBuildArtifacts@1
- displayName: Publish Graph to Artifacts
- inputs:
- PathtoPublish: '$(Build.StagingDirectory)/GraphFiles'
- PublishLocation: Container
- ArtifactName: GraphFiles
- continueOnError: true
- condition: always()
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index fe36c28e2b..01da2420df 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -24,12 +24,17 @@ parameters:
enablePublishBuildAssets: false
enablePublishTestResults: false
enablePublishUsingPipelines: false
+ disableComponentGovernance: ''
mergeTestResults: false
testRunTitle: ''
testResultsFormat: ''
name: ''
preSteps: []
runAsPublic: false
+# Sbom related params
+ enableSbom: true
+ PackageVersion: 6.0.0
+ BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
jobs:
- job: ${{ parameters.name }}
@@ -68,6 +73,10 @@ jobs:
- ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
- name: EnableRichCodeNavigation
value: 'true'
+ # Retry signature validation up to three times, waiting 2 seconds between attempts.
+ # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
+ - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
+ value: 3,2000
- ${{ each variable in parameters.variables }}:
# handle name-value variable syntax
# example:
@@ -76,7 +85,7 @@ jobs:
- ${{ if ne(variable.name, '') }}:
- name: ${{ variable.name }}
value: ${{ variable.value }}
-
+
# handle variable groups
- ${{ if ne(variable.group, '') }}:
- group: ${{ variable.group }}
@@ -114,7 +123,7 @@ jobs:
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
- - task: NuGetAuthenticate@0
+ - task: NuGetAuthenticate@1
- ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
- task: DownloadPipelineArtifact@2
@@ -136,14 +145,20 @@ jobs:
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
continueOnError: true
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}:
- - task: ComponentGovernanceComponentDetection@0
- continueOnError: true
+ - template: /eng/common/templates/steps/component-governance.yml
+ parameters:
+ ${{ if eq(parameters.disableComponentGovernance, '') }}:
+ ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+ disableComponentGovernance: false
+ ${{ else }}:
+ disableComponentGovernance: true
+ ${{ else }}:
+ disableComponentGovernance: ${{ parameters.disableComponentGovernance }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildCleanup@1
- displayName: Execute Microbuild cleanup tasks
+ displayName: Execute Microbuild cleanup tasks
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
env:
@@ -211,7 +226,7 @@ jobs:
displayName: Publish XUnit Test Results
inputs:
testResultsFormat: 'xUnit'
- testResultsFiles: '*.xml'
+ testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
mergeTestResults: ${{ parameters.mergeTestResults }}
@@ -222,7 +237,7 @@ jobs:
displayName: Publish TRX Test Results
inputs:
testResultsFormat: 'VSTest'
- testResultsFiles: '*.trx'
+ testResultsFiles: '*.trx'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
mergeTestResults: ${{ parameters.mergeTestResults }}
@@ -246,3 +261,9 @@ jobs:
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+ - template: /eng/common/templates/steps/generate-sbom.yml
+ parameters:
+ PackageVersion: ${{ parameters.packageVersion}}
+ BuildDropPath: ${{ parameters.buildDropPath }}
diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml
index 2b55a567f8..5b1b77d1c7 100644
--- a/eng/common/templates/job/onelocbuild.yml
+++ b/eng/common/templates/job/onelocbuild.yml
@@ -3,15 +3,15 @@ parameters:
dependsOn: ''
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
- pool:
- vmImage: windows-2019
-
+ pool: ''
+
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
SourcesDirectory: $(Build.SourcesDirectory)
CreatePr: true
AutoCompletePr: false
+ ReusePr: true
UseLfLineEndings: true
UseCheckedInLocProjectJson: false
LanguageSet: VS_Main_Languages
@@ -30,7 +30,18 @@ jobs:
displayName: OneLocBuild
- pool: ${{ parameters.pool }}
+ ${{ if ne(parameters.pool, '') }}:
+ pool: ${{ parameters.pool }}
+ ${{ if eq(parameters.pool, '') }}:
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
@@ -64,6 +75,8 @@ jobs:
${{ if eq(parameters.CreatePr, true) }}:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+ ${{ if eq(parameters.RepoType, 'gitHub') }}:
+ isShouldReusePrSelected: ${{ parameters.ReusePr }}
packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index fe9dfdf720..cab9056d1e 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -38,10 +38,6 @@ jobs:
value: ${{ parameters.configuration }}
- group: Publish-Build-Assets
- group: AzureDevOps-Artifact-Feeds-Pats
- # Skip component governance and codesign validation for SDL. These jobs
- # create no content.
- - name: skipComponentGovernanceDetection
- value: true
- name: runCodesignValidationInjection
value: false
@@ -55,9 +51,9 @@ jobs:
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
-
+
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - task: NuGetAuthenticate@0
+ - task: NuGetAuthenticate@1
- task: PowerShell@2
displayName: Enable cross-org NuGet feed authentication
@@ -65,20 +61,25 @@ jobs:
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Build Assets
+ condition: ${{ parameters.condition }}
+ continueOnError: ${{ parameters.continueOnError }}
inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: eng\common\sdk-task.ps1
+ arguments: >
+ -task PublishBuildAssets
+ -restore
+ -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
- /p:BuildAssetRegistryToken=$(MaestroAccessToken)
- /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+ /p:MaestroApiEndpoint=https://maestro.dot.net
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:Configuration=$(_BuildConfig)
/p:OfficialBuildId=$(Build.BuildNumber)
- condition: ${{ parameters.condition }}
- continueOnError: ${{ parameters.continueOnError }}
-
+
- task: powershell@2
displayName: Create ReleaseConfigs Artifact
inputs:
@@ -87,7 +88,7 @@ jobs:
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId)
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)"
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild)
-
+
- task: PublishBuildArtifacts@1
displayName: Publish ReleaseConfigs Artifact
inputs:
@@ -113,7 +114,7 @@ jobs:
- task: PublishBuildArtifacts@1
displayName: Publish SymbolPublishingExclusionsFile Artifact
- condition: eq(variables['SymbolExclusionFile'], 'true')
+ condition: eq(variables['SymbolExclusionFile'], 'true')
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
PublishLocation: Container
@@ -122,4 +123,4 @@ jobs:
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/templates/steps/publish-logs.yml
parameters:
- JobLabel: 'Publish_Artifacts_Logs'
+ JobLabel: 'Publish_Artifacts_Logs'
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index 5023d36dcb..8009ccb954 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -31,10 +31,11 @@ parameters:
# container and pool.
platform: {}
- # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
- # source-build builds run in Docker, including the default managed platform.
- defaultContainerHostPool:
- vmImage: ubuntu-20.04
+ # If set to true and running on a non-public project,
+ # Internal blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
@@ -47,7 +48,15 @@ jobs:
container: ${{ parameters.platform.container }}
${{ if eq(parameters.platform.pool, '') }}:
- pool: ${{ parameters.defaultContainerHostPool }}
+ # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+ # source-build builds run in Docker, including the default managed platform.
+ pool:
+ ${{ if eq(variables['System.TeamProject'], 'public') }}:
+ name: NetCore-Svc-Public
+ demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+ ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}
@@ -55,6 +64,8 @@ jobs:
clean: all
steps:
+ - ${{ if eq(parameters.enableInternalSources, true) }}:
+ - template: /eng/common/templates/steps/enable-internal-runtimes.yml
- template: /eng/common/templates/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
index 1cc0c29e4f..47f36e98bc 100644
--- a/eng/common/templates/job/source-index-stage1.yml
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -1,12 +1,13 @@
parameters:
runAsPublic: false
- sourceIndexPackageVersion: 1.0.1-20210614.1
+ sourceIndexUploadPackageVersion: 2.0.0-20240502.12
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
pool:
- vmImage: vs2017-win2016
+ vmImage: windows-2019
condition: ''
dependsOn: ''
@@ -15,14 +16,14 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- - name: SourceIndexPackageVersion
- value: ${{ parameters.sourceIndexPackageVersion }}
+ - name: SourceIndexUploadPackageVersion
+ value: ${{ parameters.sourceIndexUploadPackageVersion }}
+ - name: SourceIndexProcessBinlogPackageVersion
+ value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - group: source-dot-net stage1 variables
pool: ${{ parameters.pool }}
steps:
@@ -30,16 +31,16 @@ jobs:
- ${{ preStep }}
- task: UseDotNet@2
- displayName: Use .NET Core sdk 3.1
+ displayName: Use .NET 8 SDK
inputs:
packageType: sdk
- version: 3.1.x
+ version: 8.0.x
installationPath: $(Agent.TempDirectory)/dotnet
workingDirectory: $(Agent.TempDirectory)
- script: |
- $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
- $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
@@ -51,7 +52,21 @@ jobs:
displayName: Process Binlog into indexable sln
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
+ - task: AzureCLI@2
+ displayName: Get stage 1 auth token
+ inputs:
+ azureSubscription: 'SourceDotNet Stage1 Publish'
+ addSpnToEnvironment: true
+ scriptType: 'ps'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+ echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
+
+ - script: |
+ az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
+ displayName: "Login to Azure"
+
+ - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
displayName: Upload stage1 artifacts to source index
- env:
- BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml
new file mode 100644
index 0000000000..54c393af44
--- /dev/null
+++ b/eng/common/templates/jobs/codeql-build.yml
@@ -0,0 +1,31 @@
+parameters:
+ # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+ continueOnError: false
+ # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+ jobs: []
+ # Optional: if specified, restore and use this version of Guardian instead of the default.
+ overrideGuardianVersion: ''
+
+jobs:
+- template: /eng/common/templates/jobs/jobs.yml
+ parameters:
+ enableMicrobuild: false
+ enablePublishBuildArtifacts: false
+ enablePublishTestResults: false
+ enablePublishBuildAssets: false
+ enablePublishUsingPipelines: false
+ enableTelemetry: true
+
+ variables:
+ - group: Publish-Build-Assets
+ # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+ # sync with the packages.config file.
+ - name: DefaultGuardianVersion
+ value: 0.110.1
+ - name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+ - name: GuardianVersion
+ value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+
+ jobs: ${{ parameters.jobs }}
+
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 90015a7e5a..f70826518c 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -20,7 +20,7 @@ parameters:
enabled: false
# Optional: Include toolset dependencies in the generated graph files
includeToolset: false
-
+
# Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
jobs: []
@@ -40,7 +40,7 @@ parameters:
jobs:
- ${{ each job in parameters.jobs }}:
- template: ../job/job.yml
- parameters:
+ parameters:
# pass along parameters
${{ each parameter in parameters }}:
${{ if ne(parameter.key, 'jobs') }}:
@@ -68,7 +68,7 @@ jobs:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-
+
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
@@ -83,17 +83,15 @@ jobs:
- ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete
pool:
- vmImage: windows-2019
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ else }}:
+ name: NetCore1ESPool-Publishing-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
+
runAsPublic: ${{ parameters.runAsPublic }}
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-
- - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
- - template: ../job/generate-graph-files.yml
- parameters:
- continueOnError: ${{ parameters.continueOnError }}
- includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
- dependsOn:
- - Asset_Registry_Publish
- pool:
- vmImage: windows-2019
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
index 00aa98eb3b..e3fbd9f38e 100644
--- a/eng/common/templates/jobs/source-build.yml
+++ b/eng/common/templates/jobs/source-build.yml
@@ -14,13 +14,19 @@ parameters:
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
- container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
# one job runs on 'defaultManagedPlatform'.
platforms: []
+ # If set to true and running on a non-public project,
+ # Internal nuget and blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
jobs:
- ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
deleted file mode 100644
index 8990dfc8c8..0000000000
--- a/eng/common/templates/post-build/channels/generic-internal-channel.yml
+++ /dev/null
@@ -1,190 +0,0 @@
-parameters:
- BARBuildId: ''
- PromoteToChannelIds: ''
- artifactsPublishingAdditionalParameters: ''
- dependsOn:
- - Validate
- publishInstallersAndChecksums: true
- symbolPublishingAdditionalParameters: ''
- stageName: ''
- channelName: ''
- channelId: ''
- transportFeed: ''
- shippingFeed: ''
- symbolsFeed: ''
-
-stages:
-- stage: ${{ parameters.stageName }}
- dependsOn: ${{ parameters.dependsOn }}
- variables:
- - template: ../common-variables.yml
- displayName: ${{ parameters.channelName }} Publishing
- jobs:
- - template: ../setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-
- - job: publish_symbols
- displayName: Symbol Publishing
- dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
- variables:
- - group: DotNet-Symbol-Server-Pats
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
- displayName: Warn about v2 Arcade Publishing Usage
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: DownloadBuildArtifacts@0
- displayName: Download Build Assets
- continueOnError: true
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- downloadType: 'specific'
- itemPattern: |
- PdbArtifacts/**
- BlobArtifacts/**
- downloadPath: '$(Build.ArtifactStagingDirectory)'
- checkDownloadedFiles: true
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
- # otherwise it'll complain about accessing a private feed.
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: PowerShell@2
- displayName: Enable cross-org publishing
- inputs:
- filePath: eng\common\enable-cross-org-publishing.ps1
- arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
-
- - task: PowerShell@2
- displayName: Publish
- inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
- /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
- /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
- /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
- /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
- /p:Configuration=Release
- /p:PublishToMSDL=false
- ${{ parameters.symbolPublishingAdditionalParameters }}
-
- - template: ../../steps/publish-logs.yml
- parameters:
- StageLabel: '${{ parameters.stageName }}'
- JobLabel: 'SymbolPublishing'
-
- - job: publish_assets
- displayName: Publish Assets
- dependsOn: setupMaestroVars
- timeoutInMinutes: 120
- variables:
- - name: BARBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- - name: IsStableBuild
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
- pool:
- vmImage: 'windows-2019'
- steps:
- - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
- displayName: Warn about v2 Arcade Publishing Usage
-
- - task: DownloadBuildArtifacts@0
- displayName: Download Build Assets
- continueOnError: true
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- downloadType: 'specific'
- itemPattern: |
- PackageArtifacts/**
- BlobArtifacts/**
- AssetManifests/**
- downloadPath: '$(Build.ArtifactStagingDirectory)'
- checkDownloadedFiles: true
-
- - task: NuGetToolInstaller@1
- displayName: 'Install NuGet.exe'
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: PowerShell@2
- displayName: Enable cross-org publishing
- inputs:
- filePath: eng\common\enable-cross-org-publishing.ps1
- arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
-
- - task: PowerShell@2
- displayName: Publish Assets
- inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:PublishingInfraVersion=2
- /p:IsStableBuild=$(IsStableBuild)
- /p:IsInternalBuild=$(IsInternalBuild)
- /p:RepositoryName=$(Build.Repository.Name)
- /p:CommitSha=$(Build.SourceVersion)
- /p:NugetPath=$(NuGetExeToolPath)
- /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
- /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
- /p:BARBuildId=$(BARBuildId)
- /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
- /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
- /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
- /p:Configuration=Release
- /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
- /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
- /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
- /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
- /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
- /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
- /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
- /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
- /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:PublishToMSDL=false
- ${{ parameters.artifactsPublishingAdditionalParameters }}
-
- - template: ../../steps/publish-logs.yml
- parameters:
- StageLabel: '${{ parameters.stageName }}'
- JobLabel: 'AssetsPublishing'
-
- - template: ../../steps/add-build-to-channel.yml
- parameters:
- ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml
deleted file mode 100644
index 3220c6a4f9..0000000000
--- a/eng/common/templates/post-build/channels/generic-public-channel.yml
+++ /dev/null
@@ -1,192 +0,0 @@
-parameters:
- BARBuildId: ''
- PromoteToChannelIds: ''
- artifactsPublishingAdditionalParameters: ''
- dependsOn:
- - Validate
- publishInstallersAndChecksums: true
- symbolPublishingAdditionalParameters: ''
- stageName: ''
- channelName: ''
- channelId: ''
- transportFeed: ''
- shippingFeed: ''
- symbolsFeed: ''
- # If the channel name is empty, no links will be generated
- akaMSChannelName: ''
-
-stages:
-- stage: ${{ parameters.stageName }}
- dependsOn: ${{ parameters.dependsOn }}
- variables:
- - template: ../common-variables.yml
- displayName: ${{ parameters.channelName }} Publishing
- jobs:
- - template: ../setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-
- - job: publish_symbols
- displayName: Symbol Publishing
- dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
- variables:
- - group: DotNet-Symbol-Server-Pats
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
- displayName: Warn about v2 Arcade Publishing Usage
-
- - task: DownloadBuildArtifacts@0
- displayName: Download Build Assets
- continueOnError: true
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- downloadType: 'specific'
- itemPattern: |
- PdbArtifacts/**
- BlobArtifacts/**
- downloadPath: '$(Build.ArtifactStagingDirectory)'
- checkDownloadedFiles: true
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
- # otherwise it'll complain about accessing a private feed.
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: PowerShell@2
- displayName: Enable cross-org publishing
- inputs:
- filePath: eng\common\enable-cross-org-publishing.ps1
- arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
-
- - task: PowerShell@2
- displayName: Publish
- inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
- /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
- /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
- /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
- /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
- /p:Configuration=Release
- ${{ parameters.symbolPublishingAdditionalParameters }}
-
- - template: ../../steps/publish-logs.yml
- parameters:
- StageLabel: '${{ parameters.stageName }}'
- JobLabel: 'SymbolPublishing'
-
- - job: publish_assets
- displayName: Publish Assets
- dependsOn: setupMaestroVars
- timeoutInMinutes: 120
- variables:
- - name: BARBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- - name: IsStableBuild
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- - name: ArtifactsCategory
- value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
- pool:
- vmImage: 'windows-2019'
- steps:
- - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
- displayName: Warn about v2 Arcade Publishing Usage
-
- - task: DownloadBuildArtifacts@0
- displayName: Download Build Assets
- continueOnError: true
- inputs:
- buildType: specific
- buildVersionToDownload: specific
- project: $(AzDOProjectName)
- pipeline: $(AzDOPipelineId)
- buildId: $(AzDOBuildId)
- downloadType: 'specific'
- itemPattern: |
- PackageArtifacts/**
- BlobArtifacts/**
- AssetManifests/**
- downloadPath: '$(Build.ArtifactStagingDirectory)'
- checkDownloadedFiles: true
-
- - task: NuGetToolInstaller@1
- displayName: 'Install NuGet.exe'
-
- # This is necessary whenever we want to publish/restore to an AzDO private feed
- - task: NuGetAuthenticate@0
- displayName: 'Authenticate to AzDO Feeds'
-
- - task: PowerShell@2
- displayName: Enable cross-org publishing
- inputs:
- filePath: eng\common\enable-cross-org-publishing.ps1
- arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
-
- - task: PowerShell@2
- displayName: Publish Assets
- inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:PublishingInfraVersion=2
- /p:ArtifactsCategory=$(ArtifactsCategory)
- /p:IsStableBuild=$(IsStableBuild)
- /p:IsInternalBuild=$(IsInternalBuild)
- /p:RepositoryName=$(Build.Repository.Name)
- /p:CommitSha=$(Build.SourceVersion)
- /p:NugetPath=$(NuGetExeToolPath)
- /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
- /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
- /p:BARBuildId=$(BARBuildId)
- /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
- /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
- /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
- /p:Configuration=Release
- /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
- /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
- /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
- /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
- /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
- /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
- /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
- /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
- /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
- /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
- /p:AkaMSClientId=$(akams-client-id)
- /p:AkaMSClientSecret=$(akams-client-secret)
- ${{ parameters.artifactsPublishingAdditionalParameters }}
-
- - template: ../../steps/publish-logs.yml
- parameters:
- StageLabel: '${{ parameters.stageName }}'
- JobLabel: 'AssetsPublishing'
-
- - template: ../../steps/add-build-to-channel.yml
- parameters:
- ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml
index 956ed7eb90..7e1af5af50 100644
--- a/eng/common/templates/post-build/common-variables.yml
+++ b/eng/common/templates/post-build/common-variables.yml
@@ -2,56 +2,7 @@ variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- - group: DotNet-MSRC-Storage
- group: Publish-Build-Assets
-
- # .NET Core 3.1 Dev
- - name: PublicDevRelease_31_Channel_Id
- value: 128
-
- # .NET 5 Dev
- - name: Net_5_Dev_Channel_Id
- value: 131
-
- # .NET Eng - Validation
- - name: Net_Eng_Validation_Channel_Id
- value: 9
-
- # .NET Eng - Latest
- - name: Net_Eng_Latest_Channel_Id
- value: 2
-
- # .NET 3 Eng - Validation
- - name: NET_3_Eng_Validation_Channel_Id
- value: 390
-
- # .NET 3 Eng
- - name: NetCore_3_Tools_Channel_Id
- value: 344
-
- # .NET Core 3.0 Internal Servicing
- - name: InternalServicing_30_Channel_Id
- value: 184
-
- # .NET Core 3.0 Release
- - name: PublicRelease_30_Channel_Id
- value: 19
-
- # .NET Core 3.1 Release
- - name: PublicRelease_31_Channel_Id
- value: 129
-
- # General Testing
- - name: GeneralTesting_Channel_Id
- value: 529
-
- # .NET Core 3.1 Blazor Features
- - name: NetCore_31_Blazor_Features_Channel_Id
- value: 531
-
- # .NET Core Experimental
- - name: NetCore_Experimental_Channel_Id
- value: 562
# Whether the build is internal or not
- name: IsInternalBuild
@@ -59,9 +10,7 @@ variables:
# Default Maestro++ API Endpoint and API Version
- name: MaestroApiEndPoint
- value: "https://maestro-prod.westus2.cloudapp.azure.com"
- - name: MaestroApiAccessToken
- value: $(MaestroAccessToken)
+ value: "https://maestro.dot.net"
- name: MaestroApiVersion
value: "2020-02-20"
@@ -70,26 +19,5 @@ variables:
- name: SymbolToolVersion
value: 1.0.1
- # Feed Configurations
- # These should include the suffix "/index.json"
-
- # Default locations for Installers and checksums
- # Public Locations
- - name: ChecksumsBlobFeedUrl
- value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
- - name: InstallersBlobFeedUrl
- value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
-
- # Private Locations
- - name: InternalChecksumsBlobFeedUrl
- value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
- - name: InternalChecksumsBlobFeedKey
- value: $(dotnetclichecksumsmsrc-storage-key)
-
- - name: InternalInstallersBlobFeedUrl
- value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
- - name: InternalInstallersBlobFeedKey
- value: $(dotnetclimsrc-access-key)
-
- name: runCodesignValidationInjection
value: false
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 4f79cf0f33..c6c2da3de6 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -1,113 +1,114 @@
parameters:
- # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
- # Publishing V2 accepts optionally outlining the publishing stages - default is inline.
- # Publishing V3 DOES NOT accept inlining the publishing stages.
- publishingInfraVersion: 2
- # When set to true the publishing templates from the repo will be used
- # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline
- inline: true
-
- # Only used if inline==false. When set to true will stall the current build until
- # the Promotion Pipeline build finishes. Otherwise, the current build will continue
- # execution concurrently with the promotion build.
- waitPublishingFinish: true
-
- BARBuildId: ''
- PromoteToChannelIds: ''
-
- enableSourceLinkValidation: false
- enableSigningValidation: true
- enableSymbolValidation: false
- enableNugetValidation: true
- publishInstallersAndChecksums: true
- SDLValidationParameters:
- enable: false
- continueOnError: false
- params: ''
- artifactNames: ''
- downloadArtifacts: true
+ # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
+ # Publishing V1 is no longer supported
+ # Publishing V2 is no longer supported
+ # Publishing V3 is the default
+ - name: publishingInfraVersion
+ displayName: Which version of publishing should be used to promote the build definition?
+ type: number
+ default: 3
+ values:
+ - 3
+
+ - name: BARBuildId
+ displayName: BAR Build Id
+ type: number
+ default: 0
+
+ - name: PromoteToChannelIds
+ displayName: Channel to promote BARBuildId to
+ type: string
+ default: ''
+
+ - name: enableSourceLinkValidation
+ displayName: Enable SourceLink validation
+ type: boolean
+ default: false
+
+ - name: enableSigningValidation
+ displayName: Enable signing validation
+ type: boolean
+ default: true
+
+ - name: enableSymbolValidation
+ displayName: Enable symbol validation
+ type: boolean
+ default: false
+
+ - name: enableNugetValidation
+ displayName: Enable NuGet validation
+ type: boolean
+ default: true
+
+ - name: publishInstallersAndChecksums
+ displayName: Publish installers and checksums
+ type: boolean
+ default: true
+
+ - name: SDLValidationParameters
+ type: object
+ default:
+ enable: false
+ continueOnError: false
+ params: ''
+ artifactNames: ''
+ downloadArtifacts: true
# These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation
- symbolPublishingAdditionalParameters: ''
- artifactsPublishingAdditionalParameters: ''
- signingValidationAdditionalParameters: ''
+ - name: symbolPublishingAdditionalParameters
+ displayName: Symbol publishing additional parameters
+ type: string
+ default: ''
+
+ - name: artifactsPublishingAdditionalParameters
+ displayName: Artifact publishing additional parameters
+ type: string
+ default: ''
+
+ - name: signingValidationAdditionalParameters
+ displayName: Signing validation additional parameters
+ type: string
+ default: ''
# Which stages should finish execution before post-build stages start
- validateDependsOn:
- - build
- publishDependsOn:
- - Validate
+ - name: validateDependsOn
+ type: object
+ default:
+ - build
- # Channel ID's instantiated in this file.
- # When adding a new channel implementation the call to `check-channel-consistency.ps1`
- # needs to be updated with the new channel ID
- NetEngLatestChannelId: 2
- NetEngValidationChannelId: 9
- NetDev5ChannelId: 131
- NetDev6ChannelId: 1296
- GeneralTestingChannelId: 529
- NETCoreToolingDevChannelId: 548
- NETCoreToolingReleaseChannelId: 549
- NETInternalToolingChannelId: 551
- NETCoreExperimentalChannelId: 562
- NetEngServicesIntChannelId: 678
- NetEngServicesProdChannelId: 679
- NetCoreSDK313xxChannelId: 759
- NetCoreSDK313xxInternalChannelId: 760
- NetCoreSDK314xxChannelId: 921
- NetCoreSDK314xxInternalChannelId: 922
- VS166ChannelId: 1010
- VS167ChannelId: 1011
- VS168ChannelId: 1154
- VSMasterChannelId: 1012
- VS169ChannelId: 1473
- VS1610ChannelId: 1692
+ - name: publishDependsOn
+ type: object
+ default:
+ - Validate
stages:
-- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- stage: Validate
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate Build Assets
variables:
- template: common-variables.yml
jobs:
- - template: setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-
- - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- - job:
- displayName: Post-build Checks
- dependsOn: setupMaestroVars
- variables:
- - name: TargetChannels
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - task: PowerShell@2
- displayName: Maestro Channels Consistency
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
- arguments: -PromoteToChannels "$(TargetChannels)"
- -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
-
- job:
displayName: NuGet Validation
- dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
pool:
- vmImage: 'windows-2019'
- variables:
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
+
steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
@@ -123,24 +124,26 @@ stages:
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
- arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
+ arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- job:
displayName: Signing Validation
- dependsOn: setupMaestroVars
condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
- variables:
- - template: common-variables.yml
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
- vmImage: 'windows-2019'
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
@@ -158,7 +161,7 @@ stages:
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- - task: NuGetAuthenticate@0
+ - task: NuGetAuthenticate@1
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
@@ -185,19 +188,22 @@ stages:
- job:
displayName: SourceLink Validation
- dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
- variables:
- - template: common-variables.yml
- - name: AzDOProjectName
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- - name: AzDOPipelineId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- - name: AzDOBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
- vmImage: 'windows-2019'
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+ name: NetCore1ESPool-Svc-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
@@ -213,9 +219,9 @@ stages:
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
- arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -ExtractPath $(Agent.BuildDirectory)/Extract/
- -GHRepoName $(Build.Repository.Name)
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+ -ExtractPath $(Agent.BuildDirectory)/Extract/
+ -GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
@@ -223,367 +229,49 @@ stages:
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
- dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
-- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- - stage: publish_using_darc
- ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- dependsOn: ${{ parameters.publishDependsOn }}
- ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
- dependsOn: ${{ parameters.validateDependsOn }}
- displayName: Publish using Darc
- variables:
- - template: common-variables.yml
- jobs:
- - template: setup-maestro-vars.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
-
- - job:
- displayName: Publish Using Darc
- dependsOn: setupMaestroVars
- timeoutInMinutes: 120
- variables:
- - name: BARBuildId
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - task: PowerShell@2
- displayName: Publish Using Darc
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
- arguments: -BuildId $(BARBuildId)
- -PublishingInfraVersion ${{ parameters.PublishingInfraVersion }}
- -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
- -WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
- -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
- -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
- -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
-
-- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NetCore_Dev5_Publish'
- channelName: '.NET 5 Dev'
- akaMSChannelName: 'net5/dev'
- channelId: ${{ parameters.NetDev5ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NetCore_Dev6_Publish'
- channelName: '.NET 6 Dev'
- akaMSChannelName: 'net6/dev'
- channelId: ${{ parameters.NetDev6ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net_Eng_Latest_Publish'
- channelName: '.NET Eng - Latest'
- akaMSChannelName: 'eng/daily'
- channelId: ${{ parameters.NetEngLatestChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net_Eng_Validation_Publish'
- channelName: '.NET Eng - Validation'
- akaMSChannelName: 'eng/validation'
- channelId: ${{ parameters.NetEngValidationChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'General_Testing_Publish'
- channelName: 'General Testing'
- akaMSChannelName: 'generaltesting'
- channelId: ${{ parameters.GeneralTestingChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_Tooling_Dev_Publishing'
- channelName: '.NET Core Tooling Dev'
- channelId: ${{ parameters.NETCoreToolingDevChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_Tooling_Release_Publishing'
- channelName: '.NET Core Tooling Release'
- channelId: ${{ parameters.NETCoreToolingReleaseChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NET_Internal_Tooling_Publishing'
- channelName: '.NET Internal Tooling'
- channelId: ${{ parameters.NETInternalToolingChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_Experimental_Publishing'
- channelName: '.NET Core Experimental'
- channelId: ${{ parameters.NETCoreExperimentalChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net_Eng_Services_Int_Publish'
- channelName: '.NET Eng Services - Int'
- channelId: ${{ parameters.NetEngServicesIntChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'Net_Eng_Services_Prod_Publish'
- channelName: '.NET Eng Services - Prod'
- channelId: ${{ parameters.NetEngServicesProdChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_SDK_314xx_Publishing'
- channelName: '.NET Core SDK 3.1.4xx'
- channelId: ${{ parameters.NetCoreSDK314xxChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_SDK_314xx_Internal_Publishing'
- channelName: '.NET Core SDK 3.1.4xx Internal'
- channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_SDK_313xx_Publishing'
- channelName: '.NET Core SDK 3.1.3xx'
- channelId: ${{ parameters.NetCoreSDK313xxChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'NETCore_SDK_313xx_Internal_Publishing'
- channelName: '.NET Core SDK 3.1.3xx Internal'
- channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS16_6_Publishing'
- channelName: 'VS 16.6'
- channelId: ${{ parameters.VS166ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS16_7_Publishing'
- channelName: 'VS 16.7'
- channelId: ${{ parameters.VS167ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS16_8_Publishing'
- channelName: 'VS 16.8'
- channelId: ${{ parameters.VS168ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS_Master_Publishing'
- channelName: 'VS Master'
- channelId: ${{ parameters.VSMasterChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS_16_9_Publishing'
- channelName: 'VS 16.9'
- channelId: ${{ parameters.VS169ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-
- - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
- parameters:
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
- dependsOn: ${{ parameters.publishDependsOn }}
- publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
- stageName: 'VS_16_10_Publishing'
- channelName: 'VS 16.10'
- channelId: ${{ parameters.VS1610ChannelId }}
- transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
- shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
- symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
+- stage: publish_using_darc
+ ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.publishDependsOn }}
+ ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+ dependsOn: ${{ parameters.validateDependsOn }}
+ displayName: Publish using Darc
+ variables:
+ - template: common-variables.yml
+ jobs:
+ - job:
+ displayName: Publish Using Darc
+ timeoutInMinutes: 120
+ pool:
+ # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+ ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+ name: VSEngSS-MicroBuild2022-1ES
+ demands: Cmd
+ # If it's not devdiv, it's dnceng
+ ${{ else }}:
+ name: NetCore1ESPool-Publishing-Internal
+ demands: ImageOverride -equals windows.vs2019.amd64
+ steps:
+ - template: setup-maestro-vars.yml
+ parameters:
+ BARBuildId: ${{ parameters.BARBuildId }}
+ PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+ - task: AzureCLI@2
+ displayName: Publish Using Darc
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: >
+ -BuildId $(BARBuildId)
+ -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+ -AzdoToken '$(System.AccessToken)'
+ -WaitPublishingFinish true
+ -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+ -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml
index 4a22b2e6f6..af3a6eb12f 100644
--- a/eng/common/templates/post-build/setup-maestro-vars.yml
+++ b/eng/common/templates/post-build/setup-maestro-vars.yml
@@ -2,77 +2,69 @@ parameters:
BARBuildId: ''
PromoteToChannelIds: ''
-jobs:
-- job: setupMaestroVars
- displayName: Setup Maestro Vars
- variables:
- - template: common-variables.yml
- pool:
- vmImage: 'windows-2019'
- steps:
- - checkout: none
-
- - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
- - task: DownloadBuildArtifacts@0
- displayName: Download Release Configs
- inputs:
- buildType: current
- artifactName: ReleaseConfigs
- checkDownloadedFiles: true
-
- - task: PowerShell@2
- name: setReleaseVars
- displayName: Set Release Configs Vars
+steps:
+ - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Release Configs
inputs:
- targetType: inline
- script: |
- try {
- if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
- $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+ buildType: current
+ artifactName: ReleaseConfigs
+ checkDownloadedFiles: true
- $BarId = $Content | Select -Index 0
- $Channels = $Content | Select -Index 1
- $IsStableBuild = $Content | Select -Index 2
+ - task: AzureCLI@2
+ name: setReleaseVars
+ displayName: Set Release Configs Vars
+ inputs:
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: pscore
+ scriptLocation: inlineScript
+ inlineScript: |
+ try {
+ if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
+ $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
- $AzureDevOpsProject = $Env:System_TeamProject
- $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
- $AzureDevOpsBuildId = $Env:Build_BuildId
- }
- else {
- $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
+ $BarId = $Content | Select -Index 0
+ $Channels = $Content | Select -Index 1
+ $IsStableBuild = $Content | Select -Index 2
- $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
- $apiHeaders.Add('Accept', 'application/json')
- $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
+ $AzureDevOpsProject = $Env:System_TeamProject
+ $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
+ $AzureDevOpsBuildId = $Env:Build_BuildId
+ }
+ else {
+ . $(Build.SourcesDirectory)\eng\common\tools.ps1
+ $darc = Get-Darc
+ $buildInfo = & $darc get-build `
+ --id ${{ parameters.BARBuildId }} `
+ --extended `
+ --output-format json `
+ --ci `
+ | convertFrom-Json
- $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
-
- $BarId = $Env:BARBuildId
- $Channels = $Env:PromoteToMaestroChannels -split ","
- $Channels = $Channels -join "]["
- $Channels = "[$Channels]"
+ $BarId = ${{ parameters.BARBuildId }}
+ $Channels = $Env:PromoteToMaestroChannels -split ","
+ $Channels = $Channels -join "]["
+ $Channels = "[$Channels]"
- $IsStableBuild = $buildInfo.stable
- $AzureDevOpsProject = $buildInfo.azureDevOpsProject
- $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
- $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
- }
+ $IsStableBuild = $buildInfo.stable
+ $AzureDevOpsProject = $buildInfo.azureDevOpsProject
+ $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
+ $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
+ }
- Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
- Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
- Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+ Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId"
+ Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels"
+ Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild"
- Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
- Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
- Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
- }
- catch {
- Write-Host $_
- Write-Host $_.Exception
- Write-Host $_.ScriptStackTrace
- exit 1
- }
- env:
- MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
- BARBuildId: ${{ parameters.BARBuildId }}
- PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
+ Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject"
+ Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId"
+ Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId"
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+ }
+ env:
+ PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml
deleted file mode 100644
index da669030da..0000000000
--- a/eng/common/templates/post-build/trigger-subscription.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-parameters:
- ChannelId: 0
-
-steps:
-- task: PowerShell@2
- displayName: Triggering subscriptions
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
- arguments: -SourceRepo $(Build.Repository.Uri)
- -ChannelId ${{ parameters.ChannelId }}
- -MaestroApiAccessToken $(MaestroAccessToken)
- -MaestroApiEndPoint $(MaestroApiEndPoint)
- -MaestroApiVersion $(MaestroApiVersion)
diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml
deleted file mode 100644
index f67a210d62..0000000000
--- a/eng/common/templates/steps/add-build-to-channel.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-parameters:
- ChannelId: 0
-
-steps:
-- task: PowerShell@2
- displayName: Add Build to Channel
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1
- arguments: -BuildId $(BARBuildId)
- -ChannelId ${{ parameters.ChannelId }}
- -MaestroApiAccessToken $(MaestroApiAccessToken)
- -MaestroApiEndPoint $(MaestroApiEndPoint)
- -MaestroApiVersion $(MaestroApiVersion)
diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml
new file mode 100644
index 0000000000..12527b80ea
--- /dev/null
+++ b/eng/common/templates/steps/component-governance.yml
@@ -0,0 +1,10 @@
+parameters:
+ disableComponentGovernance: false
+
+steps:
+- ${{ if eq(parameters.disableComponentGovernance, 'true') }}:
+ - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true"
+ displayName: Set skipComponentGovernanceDetection variable
+- ${{ if ne(parameters.disableComponentGovernance, 'true') }}:
+ - task: ComponentGovernanceComponentDetection@0
+ continueOnError: true
\ No newline at end of file
diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml
new file mode 100644
index 0000000000..54dc9416c5
--- /dev/null
+++ b/eng/common/templates/steps/enable-internal-runtimes.yml
@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+ type: string
+ default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+ type: string
+ default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+ - template: /eng/common/templates/steps/get-delegation-sas.yml
+ parameters:
+ federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+ outputVariableName: ${{ parameters.outputVariableName }}
+ expiryInHours: ${{ parameters.expiryInHours }}
+ base64Encode: ${{ parameters.base64Encode }}
+ storageAccount: dotnetbuilds
+ container: internal
+ permissions: rl
diff --git a/eng/common/templates/steps/execute-codeql.yml b/eng/common/templates/steps/execute-codeql.yml
new file mode 100644
index 0000000000..3930b16302
--- /dev/null
+++ b/eng/common/templates/steps/execute-codeql.yml
@@ -0,0 +1,32 @@
+parameters:
+ # Language that should be analyzed. Defaults to csharp
+ language: csharp
+ # Build Commands
+ buildCommands: ''
+ overrideParameters: '' # Optional: to override values for parameters.
+ additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+ # Optional: if specified, restore and use this version of Guardian instead of the default.
+ overrideGuardianVersion: ''
+ # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+ # diagnosis of problems with specific tool configurations.
+ publishGuardianDirectoryToPipeline: false
+ # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+ # parameters rather than relying on YAML. It may be better to use a local script, because you can
+ # reproduce results locally without piecing together a command based on the YAML.
+ executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
+ # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+ # 'continueOnError', the parameter value is not correctly picked up.
+ # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+ # optional: determines whether to continue the build if the step errors;
+ sdlContinueOnError: false
+
+steps:
+- template: /eng/common/templates/steps/execute-sdl.yml
+ parameters:
+ overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
+ executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
+ overrideParameters: ${{ parameters.overrideParameters }}
+ additionalParameters: '${{ parameters.additionalParameters }}
+ -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")'
+ publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
+ sdlContinueOnError: ${{ parameters.sdlContinueOnError }}
\ No newline at end of file
diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml
new file mode 100644
index 0000000000..9dd5709f66
--- /dev/null
+++ b/eng/common/templates/steps/execute-sdl.yml
@@ -0,0 +1,88 @@
+parameters:
+ overrideGuardianVersion: ''
+ executeAllSdlToolsScript: ''
+ overrideParameters: ''
+ additionalParameters: ''
+ publishGuardianDirectoryToPipeline: false
+ sdlContinueOnError: false
+ condition: ''
+
+steps:
+- task: NuGetAuthenticate@1
+ inputs:
+ nuGetServiceConnections: GuardianConnect
+
+- task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+- ${{ if ne(parameters.overrideGuardianVersion, '') }}:
+ - pwsh: |
+ Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
+ . .\sdl.ps1
+ $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }}
+ Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
+ displayName: Install Guardian (Overridden)
+
+- ${{ if eq(parameters.overrideGuardianVersion, '') }}:
+ - pwsh: |
+ Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
+ . .\sdl.ps1
+ $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts
+ Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
+ displayName: Install Guardian
+
+- ${{ if ne(parameters.overrideParameters, '') }}:
+ - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
+ displayName: Execute SDL
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+ condition: ${{ parameters.condition }}
+
+- ${{ if eq(parameters.overrideParameters, '') }}:
+ - powershell: ${{ parameters.executeAllSdlToolsScript }}
+ -GuardianCliLocation $(GuardianCliLocation)
+ -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
+ -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
+ ${{ parameters.additionalParameters }}
+ displayName: Execute SDL
+ continueOnError: ${{ parameters.sdlContinueOnError }}
+ condition: ${{ parameters.condition }}
+
+- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
+ # We want to publish the Guardian results and configuration for easy diagnosis. However, the
+ # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
+ # tooling files. Some of these files are large and aren't useful during an investigation, so
+ # exclude them by simply deleting them before publishing. (As of writing, there is no documented
+ # way to selectively exclude a dir from the pipeline artifact publish task.)
+ - task: DeleteFiles@1
+ displayName: Delete Guardian dependencies to avoid uploading
+ inputs:
+ SourceFolder: $(Agent.BuildDirectory)/.gdn
+ Contents: |
+ c
+ i
+ condition: succeededOrFailed()
+
+ - publish: $(Agent.BuildDirectory)/.gdn
+ artifact: GuardianConfiguration
+ displayName: Publish GuardianConfiguration
+ condition: succeededOrFailed()
+
+ # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration
+ # with the "SARIF SAST Scans Tab" Azure DevOps extension
+ - task: CopyFiles@2
+ displayName: Copy SARIF files
+ inputs:
+ flattenFolders: true
+ sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/
+ contents: '**/*.sarif'
+ targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs
+ condition: succeededOrFailed()
+
+ # Use PublishBuildArtifacts because the SARIF extension only checks this case
+ # see microsoft/sarif-azuredevops-extension#4
+ - task: PublishBuildArtifacts@1
+ displayName: Publish SARIF files to CodeAnalysisLogs container
+ inputs:
+ pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs
+ artifactName: CodeAnalysisLogs
+ condition: succeededOrFailed()
\ No newline at end of file
diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml
new file mode 100644
index 0000000000..f4d7937f37
--- /dev/null
+++ b/eng/common/templates/steps/generate-sbom.yml
@@ -0,0 +1,44 @@
+# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
+# PackageName - The name of the package this SBOM represents.
+# PackageVersion - The version of the package this SBOM represents.
+# ManifestDirPath - The path of the directory where the generated manifest files will be placed
+
+parameters:
+ PackageVersion: 6.0.0
+ BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
+ PackageName: '.NET'
+ ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
+ sbomContinueOnError: true
+
+steps:
+- task: PowerShell@2
+ displayName: Prep for SBOM generation in (Non-linux)
+ condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
+ inputs:
+ filePath: ./eng/common/generate-sbom-prep.ps1
+ arguments: ${{parameters.manifestDirPath}}
+
+# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
+- script: |
+ chmod +x ./eng/common/generate-sbom-prep.sh
+ ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
+ displayName: Prep for SBOM generation in (Linux)
+ condition: eq(variables['Agent.Os'], 'Linux')
+ continueOnError: ${{ parameters.sbomContinueOnError }}
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+ displayName: 'Generate SBOM manifest'
+ continueOnError: ${{ parameters.sbomContinueOnError }}
+ inputs:
+ PackageName: ${{ parameters.packageName }}
+ BuildDropPath: ${{ parameters.buildDropPath }}
+ PackageVersion: ${{ parameters.packageVersion }}
+ ManifestDirPath: ${{ parameters.manifestDirPath }}
+
+- task: PublishPipelineArtifact@1
+ displayName: Publish SBOM manifest
+ continueOnError: ${{parameters.sbomContinueOnError}}
+ inputs:
+ targetPath: '${{parameters.manifestDirPath}}'
+ artifactName: $(ARTIFACT_NAME)
+
diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml
new file mode 100644
index 0000000000..c0e8f91317
--- /dev/null
+++ b/eng/common/templates/steps/get-delegation-sas.yml
@@ -0,0 +1,43 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: false
+- name: storageAccount
+ type: string
+- name: container
+ type: string
+- name: permissions
+ type: string
+ default: 'rl'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ # Calculate the expiration of the SAS token and convert to UTC
+ $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+
+ if ('${{ parameters.base64Encode }}' -eq 'true') {
+ $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+ }
+
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml
new file mode 100644
index 0000000000..c8c49cc0e8
--- /dev/null
+++ b/eng/common/templates/steps/get-federated-access-token.yml
@@ -0,0 +1,28 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+ type: string
+ default: '499b84ac-1321-427f-aa17-267ca6975798'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Getting federated access token for feeds'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+ exit 1
+ }
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken"
\ No newline at end of file
diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml
new file mode 100644
index 0000000000..83d97a26a0
--- /dev/null
+++ b/eng/common/templates/steps/retain-build.yml
@@ -0,0 +1,28 @@
+parameters:
+ # Optional azure devops PAT with build execute permissions for the build's organization,
+ # only needed if the build that should be retained ran on a different organization than
+ # the pipeline where this template is executing from
+ Token: ''
+ # Optional BuildId to retain, defaults to the current running build
+ BuildId: ''
+ # Azure devops Organization URI for the build in the https://dev.azure.com/ format.
+ # Defaults to the organization the current pipeline is running on
+ AzdoOrgUri: '$(System.CollectionUri)'
+ # Azure devops project for the build. Defaults to the project the current pipeline is running on
+ AzdoProject: '$(System.TeamProject)'
+
+steps:
+ - task: powershell@2
+ inputs:
+ targetType: 'filePath'
+ filePath: eng/common/retain-build.ps1
+ pwsh: true
+ arguments: >
+ -AzdoOrgUri: ${{parameters.AzdoOrgUri}}
+ -AzdoProject ${{parameters.AzdoProject}}
+ -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }}
+ -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}}
+ displayName: Enable permanent build retention
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ BUILD_ID: $(Build.BuildId)
\ No newline at end of file
diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml
index ba40dc82f1..b5b3e5aeb3 100644
--- a/eng/common/templates/steps/source-build.yml
+++ b/eng/common/templates/steps/source-build.yml
@@ -43,8 +43,8 @@ steps:
# In that case, add variables to allow the download of internal runtimes if the specified versions are not found
# in the default public locations.
internalRuntimeDownloadArgs=
- if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
- internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
+ if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
+ internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
fi
buildConfig=Release
@@ -63,6 +63,11 @@ steps:
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi
+ runtimeOsArgs=
+ if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
+ runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
+ fi
+
publishArgs=
if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
publishArgs='--publish'
@@ -75,6 +80,7 @@ steps:
$internalRuntimeDownloadArgs \
$internalRestoreArgs \
$targetRidArgs \
+ $runtimeOsArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true
displayName: Build
diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml
new file mode 100644
index 0000000000..1a860bd040
--- /dev/null
+++ b/eng/common/templates/variables/sdl-variables.yml
@@ -0,0 +1,7 @@
+variables:
+# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+# sync with the packages.config file.
+- name: DefaultGuardianVersion
+ value: 0.110.1
+- name: GuardianPackagesConfigFile
+ value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
\ No newline at end of file
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 4448428994..81d7b0355e 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Disable telemetry on CI.
if ($ci) {
$env:DOTNET_CLI_TELEMETRY_OPTOUT=1
-
- # In case of network error, try to log the current IP for reference
- Try-LogClientIpAddress
}
# Source Build uses DotNetCoreSdkDir variable
@@ -301,31 +298,44 @@ function InstallDotNet([string] $dotnetRoot,
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
if ($noPath) { $installParameters.NoPath = $True }
- try {
- & $installScript @installParameters
- }
- catch {
- if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
- Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
- if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+ $variations = @()
+ $variations += @($installParameters)
- if ($runtimeSourceFeedKey) {
- $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
- $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
- $installParameters.FeedCredential = $decodedString
- }
+ $dotnetBuilds = $installParameters.Clone()
+ $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public"
+ $variations += @($dotnetBuilds)
- try {
- & $installScript @installParameters
- }
- catch {
- Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
- ExitWithExitCode 1
- }
+ if ($runtimeSourceFeed) {
+ $runtimeSource = $installParameters.Clone()
+ $runtimeSource.AzureFeed = $runtimeSourceFeed
+ if ($runtimeSourceFeedKey) {
+ $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+ $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+ $runtimeSource.FeedCredential = $decodedString
+ }
+ $variations += @($runtimeSource)
+ }
+
+ $installSuccess = $false
+ foreach ($variation in $variations) {
+ if ($variation | Get-Member AzureFeed) {
+ $location = $variation.AzureFeed
} else {
- Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
- ExitWithExitCode 1
+ $location = "public location";
}
+ Write-Host "Attempting to install dotnet from $location."
+ try {
+ & $installScript @variation
+ $installSuccess = $true
+ break
+ }
+ catch {
+ Write-Host "Failed to install dotnet from $location."
+ }
+ }
+ if (-not $installSuccess) {
+ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
+ ExitWithExitCode 1
}
}
@@ -625,6 +635,10 @@ function InitializeNativeTools() {
InstallDirectory = "$ToolsDir"
}
}
+ if ($env:NativeToolsOnMachine) {
+ Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..."
+ $nativeArgs += @{ PathPromotion = $true }
+ }
& "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
}
}
@@ -882,24 +896,6 @@ if (!$disableConfigureToolsetImport) {
}
}
-function Try-LogClientIpAddress()
-{
- Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
- try
- {
- $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
- $lines = $result.Content.Split([Environment]::NewLine)
- $socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
- Write-Host $socketIp
- $clientIp = $lines | Select-String -Pattern "^Client IP:.*"
- Write-Host $clientIp
- }
- catch
- {
- Write-Host "Unable to get this machine's effective IP address for logging: $_"
- }
-}
-
#
# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
#
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 6a4871ef72..e555c34269 100644
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -188,28 +188,29 @@ function InstallDotNet {
GetDotNetInstallScript "$root"
local install_script=$_GetDotNetInstallScript
- local archArg=''
+ local installParameters=(--version $version --install-dir "$root")
+
if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
- archArg="--architecture $3"
+ installParameters+=(--architecture $3)
fi
- local runtimeArg=''
if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
- runtimeArg="--runtime $4"
+ installParameters+=(--runtime $4)
fi
- local skipNonVersionedFilesArg=""
if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
- skipNonVersionedFilesArg="--skip-non-versioned-files"
+ installParameters+=(--skip-non-versioned-files)
fi
- bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
- local exit_code=$?
- echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."
- local runtimeSourceFeed=''
- if [[ -n "${6:-}" ]]; then
- runtimeSourceFeed="--azure-feed $6"
- fi
+ local variations=() # list of variable names with parameter arrays in them
+
+ local public_location=("${installParameters[@]}")
+ variations+=(public_location)
- local runtimeSourceFeedKey=''
+ local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public")
+ variations+=(dotnetbuilds)
+
+ if [[ -n "${6:-}" ]]; then
+ variations+=(private_feed)
+ local private_feed=("${installParameters[@]}" --azure-feed $6)
if [[ -n "${7:-}" ]]; then
# The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
# '-d'. To work around this, do a simple detection and switch the parameter
@@ -219,22 +220,27 @@ function InstallDotNet {
decodeArg="-d"
fi
decodedFeedKey=`echo $7 | base64 $decodeArg`
- runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+ private_feed+=(--feed-credential $decodedFeedKey)
fi
+ fi
- if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
- bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
- local exit_code=$?
- Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
- ExitWithExitCode $exit_code
- }
- else
- if [[ $exit_code != 0 ]]; then
- Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
- fi
- ExitWithExitCode $exit_code
+ local installSuccess=0
+ for variationName in "${variations[@]}"; do
+ local name="$variationName[@]"
+ local variation=("${!name}")
+ echo "Attempting to install dotnet from $variationName."
+ bash "$install_script" "${variation[@]}" && installSuccess=1
+ if [[ "$installSuccess" -eq 1 ]]; then
+ break
fi
- }
+
+ echo "Failed to install dotnet from $variationName."
+ done
+
+ if [[ "$installSuccess" -eq 0 ]]; then
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations."
+ ExitWithExitCode 1
+ fi
}
function with_retries {
@@ -399,13 +405,6 @@ function StopProcesses {
return 0
}
-function TryLogClientIpAddress () {
- echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes'
- if command -v curl > /dev/null; then
- curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true
- fi
-}
-
function MSBuild {
local args=$@
if [[ "$pipelines_log" == true ]]; then
diff --git a/global.json b/global.json
index 2f95f6537a..3b7e821333 100644
--- a/global.json
+++ b/global.json
@@ -1,6 +1,6 @@
{
"tools": {
- "dotnet": "6.0.100-rc.1.21430.12",
+ "dotnet": "6.0.133",
"runtimes": {
"dotnet": [
"3.1.7"
@@ -12,11 +12,11 @@
"xcopy-msbuild": "16.10.0-preview2"
},
"sdk": {
- "version": "6.0.100-rc.1.21430.12",
+ "version": "6.0.133",
"allowPrerelease": true,
"rollForward": "patch"
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21519.3"
+ "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24459.5"
}
}