diff --git a/.azure/pipelines/ci-public.yml b/.azure/pipelines/ci-public.yml new file mode 100644 index 000000000000..eb9ffeaa3cd5 --- /dev/null +++ b/.azure/pipelines/ci-public.yml @@ -0,0 +1,621 @@ +# +# See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for details on this file. +# + +# Configure which branches trigger builds +trigger: + batch: true + branches: + include: + - main + - release/* + +# Run PR validation on all branches +# This doesn't have any path exclusions, even for things like docs, because +# we have it configured in GitHub as a required check, and for it to pass +# it must actually run, even if it's not relevant to a particular change. +pr: + autoCancel: true + branches: + include: + - '*' + +parameters: +# Choose whether to skip tests when running pipeline manually. +- name: skipTests + default: false + displayName: Skip tests? + type: boolean +# Parameters below are ignored in public builds. +# +# Choose whether to enable binlogs when running pipeline manually. +# Binary logs are enabled by default in public builds and aren't designed to be disabled there. +- name: produceBinlogs + default: false + displayName: Produce binlogs? + type: boolean + +variables: +- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: true +- name: _TeamName + value: AspNetCore +- name: _PublishUsingPipelines + value: true +- ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: + - name: PostBuildSign + value: false +- ${{ else }}: + - name: PostBuildSign + value: true +- name: _UseHelixOpenQueues + value: 'true' +- name: _BuildArgs + value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' +- name: _PublishArgs + value: '' +# Write binary logs for all main Windows build steps except the x86 one in public and PR builds. +- name: WindowsArm64LogArgs + value: /bl:artifacts/log/Release/Build.arm64.binlog +- name: Windows64LogArgs + value: /bl:artifacts/log/Release/Build.x64.binlog +- name: Windows86LogArgs + value: -ExcludeCIBinaryLog +- name: WindowsSignLogArgs + value: /bl:artifacts/log/Release/Build.CodeSign.binlog +- name: WindowsInstallersLogArgs + value: /bl:artifacts/log/Release/Build.Installers.binlog +- name: WindowsArm64InstallersLogArgs + value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog +- name: _SignType + value: '' +- name: _InternalRuntimeDownloadArgs + value: '' +- name: _InternalRuntimeDownloadCodeSignArgs + value: '' +- name: Codeql.Enabled + value: false +- name: Codeql.SkipTaskAutoInjection + value: true +- template: /eng/common/templates/variables/pool-providers.yml + +stages: +- stage: build + displayName: Build + jobs: + # Code check + - template: jobs/default-build.yml + parameters: + jobName: Code_check + jobDisplayName: Code check + agentOs: Windows + steps: + - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) + displayName: Run eng/scripts/CodeCheck.ps1 + artifacts: + - name: Code_Check_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Build Windows (x64/x86/arm64) + - template: jobs/default-build.yml + parameters: + codeSign: true + jobName: Windows_build + jobDisplayName: "Build: Windows x64/x86/arm64" + agentOs: Windows + steps: + - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: + - script: "echo ##vso[build.addbuildtag]daily-build" + displayName: 'Set CI daily-build tag' + + # !!! NOTE !!! Some of these steps have disabled code signing. + # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even + # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. + # The sign settings have been configured to + - script: ./eng/build.cmd + -ci + -arch x64 + -pack + -all + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows64LogArgs) + displayName: Build x64 + + # Build the x86 shared framework + # This is going to actually build x86 native assets. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch x86 + -pack + -all + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows86LogArgs) + displayName: Build x86 + + # Build the arm64 shared framework + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -pack + -noBuildJava + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64LogArgs) + displayName: Build ARM64 + + # Submit a manual build (in public or internal project) to validate changes to site extensions. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - script: .\src\SiteExtensions\build.cmd + -ci + -noBuildRepoTasks + -pack + -noBuildDeps + -noBuildNative + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build SiteExtension + + # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If + # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other + # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -noBuildNative + -noBuild + -sign + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(WindowsSignLogArgs) + displayName: Code sign packages + + # Windows installers bundle x86/x64/arm64 assets + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsInstallersLogArgs) + displayName: Build Installers + + # Windows installers bundle and sharedfx msi for arm64 + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:AssetManifestFileName=aspnetcore-win.xml + $(_BuildArgs) + $(_PublishArgs) + /p:PublishInstallerBaseVersion=true + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64InstallersLogArgs) + displayName: Build ARM64 Installers + + artifacts: + - name: Windows_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Packages + path: artifacts/packages/ + - name: Windows_HostingBundle + path: artifacts/bin/WindowsHostingBundle + - name: Windows_ANCM_Msi + path: artifacts/bin/ANCMv2 + - name: Windows_ANCMIISExpress_Msi + path: artifacts/bin/AncmIISExpressV2 + + # Build MacOS arm64 + - template: jobs/default-build.yml + parameters: + jobName: MacOs_arm64_build + jobDisplayName: "Build: macOS arm64" + agentOs: macOs + timeoutInMinutes: 90 + buildArgs: + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: MacOS_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: MacOS_arm64 + + # Build MacOS x64 + - template: jobs/default-build.yml + parameters: + jobName: MacOs_x64_build + jobDisplayName: "Build: macOS x64" + agentOs: macOs + timeoutInMinutes: 90 + buildArgs: + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: MacOS_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: MacOS_x64 + + # Build Linux x64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_x64_build + jobDisplayName: "Build: Linux x64" + agentOs: Linux + useHostedUbuntu: false + steps: + - script: ./eng/build.sh + --ci + --arch x64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build Debian installers + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false + artifacts: + - name: Linux_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_x64 + + # Build Linux ARM + - template: jobs/default-build.yml + parameters: + jobName: Linux_arm_build + jobDisplayName: "Build: Linux ARM" + agentOs: Linux + buildArgs: + --arch arm + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_arm_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_arm + + # Build Linux ARM64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_arm64_build + jobDisplayName: "Build: Linux ARM64" + agentOs: Linux + steps: + - script: ./eng/build.sh + --ci + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false + artifacts: + - name: Linux_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_arm64 + + # Build Linux Musl x64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_x64_build + jobDisplayName: "Build: Linux Musl x64" + agentOs: Linux + container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + buildArgs: + --arch x64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + disableComponentGovernance: true + artifacts: + - name: Linux_musl_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_musl_x64 + + # Build Linux Musl ARM + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_arm_build + jobDisplayName: "Build: Linux Musl ARM" + agentOs: Linux + useHostedUbuntu: false + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + buildArgs: + --arch arm + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_musl_arm_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_musl_arm + + # Build Linux Musl ARM64 + - template: jobs/default-build.yml + parameters: + jobName: Linux_musl_arm64_build + jobDisplayName: "Build: Linux Musl ARM64" + agentOs: Linux + useHostedUbuntu: false + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine + buildArgs: + --arch arm64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_musl_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_musl_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: jobs/codesign-xplat.yml + parameters: + inputName: Linux_musl_arm64 + + - ${{ if ne(parameters.skipTests, 'true') }}: + # Test jobs + - template: jobs/default-build.yml + parameters: + jobName: Windows_Test + jobDisplayName: "Test: Windows Server x64" + agentOs: Windows + isAzDOTestingJob: true + # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. + cancelTimeoutInMinutes: 30 + buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true + /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true + $(_InternalRuntimeDownloadArgs) + beforeBuild: + - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" + displayName: Setup IISExpress test certificates and schema + artifacts: + - name: Windows_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: jobs/default-build.yml + parameters: + jobName: MacOS_Test + jobDisplayName: "Test: macOS" + agentOs: macOS + timeoutInMinutes: 90 + isAzDOTestingJob: true + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-mac.sh" + displayName: Installing Nginx + artifacts: + - name: MacOS_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: jobs/default-build.yml + parameters: + jobName: Linux_Test + jobDisplayName: "Test: Ubuntu x64" + agentOs: Linux + isAzDOTestingJob: true + useHostedUbuntu: false + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-linux.sh" + displayName: Installing Nginx + - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" + displayName: Increase inotify limit + artifacts: + - name: Linux_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + # Helix x64 + - template: jobs/default-build.yml + parameters: + jobName: Helix_x64 + jobDisplayName: 'Tests: Helix x64' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework + - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Source build + - template: /eng/common/templates/job/source-build.yml + parameters: + platform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' + skipPublishValidation: true + jobProperties: + timeoutInMinutes: 120 + variables: + # Log environment variables in binary logs to ease debugging + MSBUILDLOGALLENVIRONMENTVARIABLES: true diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index b184f7cef685..2b65d7e703b6 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -21,17 +21,6 @@ pr: include: - '*' -schedules: -- cron: 0 9 * * 1 - displayName: "Run CodeQL3000 weekly, Monday at 2:00 AM PDT" - branches: - include: - - release/2.1 - - release/6.0 - - release/7.0 - - main - always: true - parameters: # Choose whether to skip tests when running pipeline manually. - name: skipTests @@ -40,12 +29,6 @@ parameters: type: boolean # Parameters below are ignored in public builds. # -# Choose whether to run the CodeQL3000 tasks. -# Manual builds align w/ official builds unless this parameter is true. -- name: runCodeQL3000 - default: false - displayName: Run CodeQL3000 tasks - type: boolean # Choose whether to enable binlogs when running pipeline manually. # Binary logs are enabled by default in public builds and aren't designed to be disabled there. - name: produceBinlogs @@ -68,10 +51,10 @@ variables: value: true - name: _UseHelixOpenQueues value: ${{ ne(variables['System.TeamProject'], 'internal') }} -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: +- ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - name: enableSourceIndex value: true -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - name: _BuildArgs value: /p:TeamName=$(_TeamName) /p:OfficialBuildId=$(Build.BuildNumber) @@ -99,12 +82,12 @@ variables: value: -ExcludeCIBinaryLog - name: WindowsArm64InstallersLogArgs value: -ExcludeCIBinaryLog -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - name: _BuildArgs value: '/p:SkipTestBuild=true /p:PostBuildSign=$(PostBuildSign)' - name: _PublishArgs value: '' -- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest'), eq(parameters.produceBinlogs, 'true')) }}: +- ${{ if or(in(variables['Build.Reason'], 'PullRequest'), eq(parameters.produceBinlogs, 'true')) }}: # Write binary logs for all main Windows build steps except the x86 one in public and PR builds. - name: WindowsArm64LogArgs value: /bl:artifacts/log/Release/Build.arm64.binlog @@ -118,689 +101,653 @@ variables: value: /bl:artifacts/log/Release/Build.Installers.binlog - name: WindowsArm64InstallersLogArgs value: /bl:artifacts/log/Release/Build.Installers.Arm64.binlog -- ${{ if ne(variables['System.TeamProject'], 'internal') }}: +- group: DotNetBuilds storage account read tokens +- name: _InternalRuntimeDownloadArgs + value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal + -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) + /p:DotNetAssetRootAccessTokenSuffix='$(dotnetbuilds-internal-container-read-token-base64)' +# The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This +# is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically. +- name: _InternalRuntimeDownloadCodeSignArgs + value: $(_InternalRuntimeDownloadArgs) + /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) +- group: DotNet-HelixApi-Access +- ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - name: _SignType - value: '' - - name: _InternalRuntimeDownloadArgs - value: '' - - name: _InternalRuntimeDownloadCodeSignArgs - value: '' -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: -RuntimeSourceFeed https://dotnetbuilds.blob.core.windows.net/internal - -RuntimeSourceFeedKey $(dotnetbuilds-internal-container-read-token-base64) - /p:DotNetAssetRootAccessTokenSuffix='$(dotnetbuilds-internal-container-read-token-base64)' - # The code signing doesn't use the aspnet build scripts, so the msbuild parameters have to be passed directly. This - # is awkward but necessary because the eng/common/ build scripts don't add the msbuild properties automatically. - - name: _InternalRuntimeDownloadCodeSignArgs - value: $(_InternalRuntimeDownloadArgs) - /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - group: DotNet-HelixApi-Access - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: real - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: _SignType - value: test -- name: runCodeQL3000 - value: ${{ or(eq(variables['Build.Reason'], 'Schedule'), and(eq(variables['Build.Reason'], 'Manual'), eq(parameters.runCodeQL3000, 'true'))) }} -- template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: build - displayName: Build - jobs: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), eq(variables.runCodeQL3000, 'true')) }}: - - template: jobs/default-build.yml - parameters: - jobName: build - jobDisplayName: Build and run CodeQL3000 - agentOs: Windows - codeSign: false - # Component governance and SBOM creation are not needed here. Disable what Arcade would inject. - disableComponentGovernance: true - enableSbom: false - variables: - # Security analysis is included in normal runs. Disable its auto-injection. - - skipNugetSecurityAnalysis: true - # Do not let CodeQL3000 Extension gate scan frequency. - - Codeql.Cadence: 0 - # Enable CodeQL3000 unconditionally so it may be run on any branch. - - Codeql.Enabled: true - # Ignore the small amount of infrastructure Python code in this repo. - - Codeql.Language: cpp,csharp,java,javascript - - Codeql.ExcludePathPatterns: submodules - # Ignore test and infrastructure code. - - Codeql.SourceRoot: src - # CodeQL3000 needs this plumbed along as a variable to enable TSA. - - Codeql.TSAEnabled: ${{ eq(variables['Build.Reason'], 'Schedule') }} - # Default expects tsaoptions.json under SourceRoot. - - Codeql.TSAOptionsPath: '$(Build.SourcesDirectory)/.config/tsaoptions.json' - beforeBuild: - - task: CodeQL3000Init@0 - displayName: CodeQL Initialize - - script: "echo ##vso[build.addbuildtag]CodeQL3000" - displayName: 'Set CI CodeQL3000 tag' - condition: ne(variables.CODEQL_DIST,'') - steps: - - script: ./eng/build.cmd - -ci - -arch x64 - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - /p:UseSharedCompilation=false - displayName: Build x64 - afterBuild: - - task: CodeQL3000Finalize@0 - displayName: CodeQL Finalize - artifacts: - - name: Build_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - - ${{ else }}: # regular build - # Code check - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: - - template: jobs/default-build.yml + value: real +- ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - name: _SignType + value: test +- template: /eng/common/templates-official/variables/pool-providers.yml@self + +resources: + repositories: + # Repo: 1ESPipelineTemplates/1ESPipelineTemplates + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt + os: windows + spotBugs: + enabled: false + containers: + alpine319WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + mariner20CrossArmAlpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + mariner20CrossArm64Alpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine + stages: + - stage: build + displayName: Build + jobs: + # Code check + - ${{ if in(variables['Build.Reason'], 'PullRequest', 'Manual') }}: + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Code_check + jobDisplayName: Code check + agentOs: Windows + steps: + - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) + displayName: Run eng/scripts/CodeCheck.ps1 + artifacts: + - name: Code_Check_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Build Windows (x64/x86/arm64) + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Code_check - jobDisplayName: Code check + codeSign: true + jobName: Windows_build + jobDisplayName: "Build: Windows x64/x86/arm64" agentOs: Windows steps: - - powershell: ./eng/scripts/CodeCheck.ps1 -ci $(_InternalRuntimeDownloadArgs) - displayName: Run eng/scripts/CodeCheck.ps1 - artifacts: - - name: Code_Check_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true + - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: + - script: "echo ##vso[build.addbuildtag]daily-build" + displayName: 'Set CI daily-build tag' + + # !!! NOTE !!! Some of these steps have disabled code signing. + # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even + # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. + # The sign settings have been configured to + - script: ./eng/build.cmd + -ci + -arch x64 + -pack + -all + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(Windows64LogArgs) + displayName: Build x64 - # Build Windows (x64/x86/arm64) - - template: jobs/default-build.yml - parameters: - codeSign: true - jobName: Windows_build - jobDisplayName: "Build: Windows x64/x86/arm64" - agentOs: Windows - steps: - - ${{ if notIn(variables['Build.Reason'], 'PullRequest') }}: - - script: "echo ##vso[build.addbuildtag]daily-build" - displayName: 'Set CI daily-build tag' - - # !!! NOTE !!! Some of these steps have disabled code signing. - # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even - # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. - # The sign settings have been configured to - - script: ./eng/build.cmd - -ci - -arch x64 - -pack - -all - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows64LogArgs) - displayName: Build x64 - - # Build the x86 shared framework - # This is going to actually build x86 native assets. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch x86 - -pack - -all - -noBuildJava - -noBuildNative - /p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(Windows86LogArgs) - displayName: Build x86 - - # Build the arm64 shared framework - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch arm64 - -sign - -pack - -noBuildJava - -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(WindowsArm64LogArgs) - displayName: Build ARM64 - - # Submit a manual build (in public or internal project) to validate changes to site extensions. - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - script: .\src\SiteExtensions\build.cmd + # Build the x86 shared framework + # This is going to actually build x86 native assets. + - script: ./eng/build.cmd -ci -noBuildRepoTasks + -arch x86 -pack - -noBuildDeps + -all + -noBuildJava -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true $(_BuildArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build SiteExtension - - # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If - # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other - # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -noBuildNative - -noBuild - -sign - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(WindowsSignLogArgs) - displayName: Code sign packages - - # Windows installers bundle x86/x64/arm64 assets - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - $(WindowsInstallersLogArgs) - displayName: Build Installers - - # Windows installers bundle and sharedfx msi for arm64 - - script: ./eng/build.cmd - -ci - -noBuildRepoTasks - -arch arm64 - -sign - -buildInstallers - -noBuildNative - /p:DotNetSignType=$(_SignType) - /p:AssetManifestFileName=aspnetcore-win.xml - $(_BuildArgs) - $(_PublishArgs) - /p:PublishInstallerBaseVersion=true - $(_InternalRuntimeDownloadArgs) - $(WindowsArm64InstallersLogArgs) - displayName: Build ARM64 Installers - - artifacts: - - name: Windows_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Windows_Packages - path: artifacts/packages/ - - name: Windows_HostingBundle - path: artifacts/bin/WindowsHostingBundle - - name: Windows_ANCM_Msi - path: artifacts/bin/ANCMv2 - - name: Windows_ANCMIISExpress_Msi - path: artifacts/bin/AncmIISExpressV2 - - # Build MacOS arm64 - - template: jobs/default-build.yml - parameters: - jobName: MacOs_arm64_build - jobDisplayName: "Build: macOS arm64" - agentOs: macOs - buildArgs: - --arch arm64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: MacOS_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: MacOS_arm64 - - # Build MacOS x64 - - template: jobs/default-build.yml - parameters: - jobName: MacOs_x64_build - jobDisplayName: "Build: macOS x64" - agentOs: macOs - buildArgs: - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: MacOS_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: MacOS_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml + $(Windows86LogArgs) + displayName: Build x86 + + # Build the arm64 shared framework + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -pack + -noBuildJava + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64LogArgs) + displayName: Build ARM64 + + # Submit a manual build (in public or internal project) to validate changes to site extensions. + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - script: .\src\SiteExtensions\build.cmd + -ci + -noBuildRepoTasks + -pack + -noBuildDeps + -noBuildNative + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build SiteExtension + + # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If + # https://github.com/dotnet/arcade/issues/1957 is resolved, consider running code-signing inline with the other + # previous steps. Sign check is disabled because it is run in a separate step below, after installers are built. + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -noBuildNative + -noBuild + -sign + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(WindowsSignLogArgs) + displayName: Code sign packages + + # Windows installers bundle x86/x64/arm64 assets + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + $(WindowsInstallersLogArgs) + displayName: Build Installers + + # Windows installers bundle and sharedfx msi for arm64 + - script: ./eng/build.cmd + -ci + -noBuildRepoTasks + -arch arm64 + -sign + -buildInstallers + -noBuildNative + /p:DotNetSignType=$(_SignType) + /p:AssetManifestFileName=aspnetcore-win.xml + $(_BuildArgs) + $(_PublishArgs) + /p:PublishInstallerBaseVersion=true + $(_InternalRuntimeDownloadArgs) + $(WindowsArm64InstallersLogArgs) + displayName: Build ARM64 Installers + + artifacts: + - name: Windows_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Packages + path: artifacts/packages/ + - name: Windows_HostingBundle + path: artifacts/bin/WindowsHostingBundle + - name: Windows_ANCM_Msi + path: artifacts/bin/ANCMv2 + - name: Windows_ANCMIISExpress_Msi + path: artifacts/bin/AncmIISExpressV2 + + # Build MacOS arm64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - inputName: MacOS_x64 - - # Build Linux x64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_x64_build - jobDisplayName: "Build: Linux x64" - agentOs: Linux - useHostedUbuntu: false - steps: - - script: ./eng/build.sh - --ci - --arch x64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Build Debian installers - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + jobName: MacOs_arm64_build + jobDisplayName: "Build: macOS arm64" + agentOs: macOs + buildArgs: + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_arm64.xml $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers - installNodeJs: false - artifacts: - - name: Linux_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_x64 - - # Build Linux ARM - - template: jobs/default-build.yml - parameters: - jobName: Linux_arm_build - jobDisplayName: "Build: Linux ARM" - agentOs: Linux - buildArgs: - --arch arm - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_arm_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml + installNodeJs: false + artifacts: + - name: MacOS_arm64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: MacOS_arm64 + + # Build MacOS x64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - inputName: Linux_arm - - # Build Linux ARM64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_arm64_build - jobDisplayName: "Build: Linux ARM64" - agentOs: Linux - steps: - - script: ./eng/build.sh - --ci - --arch arm64 - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - $(_BuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Run build.sh - - script: git clean -xfd src/**/obj/; - ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs - -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm - -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + jobName: MacOs_x64_build + jobDisplayName: "Build: macOS x64" + agentOs: macOs + buildArgs: + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml $(_BuildArgs) $(_PublishArgs) $(_InternalRuntimeDownloadArgs) - displayName: Build RPM installers - installNodeJs: false - artifacts: - - name: Linux_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_arm64 - - # Build Linux Musl x64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_x64_build - jobDisplayName: "Build: Linux Musl x64" - agentOs: Linux - container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-WithNode - buildArgs: - --arch x64 - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - disableComponentGovernance: true - artifacts: - - name: Linux_musl_x64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_x64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_x64 - - # Build Linux Musl ARM - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_arm_build - jobDisplayName: "Build: Linux Musl ARM" - agentOs: Linux - useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine - buildArgs: - --arch arm - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_musl_arm_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_arm_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm - - # Build Linux Musl ARM64 - - template: jobs/default-build.yml - parameters: - jobName: Linux_musl_arm64_build - jobDisplayName: "Build: Linux Musl ARM64" - agentOs: Linux - useHostedUbuntu: false - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine - buildArgs: - --arch arm64 - --os-name linux-musl - --pack - --all - --no-build-nodejs - --no-build-java - -p:OnlyPackPlatformSpecificPackages=true - -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml - $(_BuildArgs) - $(_PublishArgs) - $(_InternalRuntimeDownloadArgs) - installNodeJs: false - artifacts: - - name: Linux_musl_arm64_Logs - path: artifacts/log/ - publishOnError: true - includeForks: true - - name: Linux_musl_arm64_Packages - path: artifacts/packages/ - - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - template: jobs/codesign-xplat.yml - parameters: - inputName: Linux_musl_arm64 + installNodeJs: false + artifacts: + - name: MacOS_x64_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_x64_Packages + path: artifacts/packages/ - - ${{ if and(ne(parameters.skipTests, 'true'), or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest', 'Manual'))) }}: - # Test jobs - - template: jobs/default-build.yml + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: MacOS_x64 + + # Build Linux x64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Windows_Test - jobDisplayName: "Test: Windows Server x64" - agentOs: Windows - isAzDOTestingJob: true - # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. - cancelTimeoutInMinutes: 30 - buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true - /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true - $(_InternalRuntimeDownloadArgs) - beforeBuild: - - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" - displayName: Setup IISExpress test certificates and schema + jobName: Linux_x64_build + jobDisplayName: "Build: Linux x64" + agentOs: Linux + useHostedUbuntu: false + steps: + - script: ./eng/build.sh + --ci + --arch x64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh bionic --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=deb + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build Debian installers + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch x64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false artifacts: - - name: Windows_Test_Logs + - name: Linux_x64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Windows_Test_Results - path: artifacts/TestResults/ + - name: Linux_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_x64 + + # Build Linux ARM + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_arm_build + jobDisplayName: "Build: Linux ARM" + agentOs: Linux + buildArgs: + --arch arm + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + artifacts: + - name: Linux_arm_Logs + path: artifacts/log/ publishOnError: true includeForks: true + - name: Linux_arm_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_arm - - template: jobs/default-build.yml + # Build Linux ARM64 + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: MacOS_Test - jobDisplayName: "Test: macOS" - agentOs: macOS - timeoutInMinutes: 240 - isAzDOTestingJob: true - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-mac.sh" - displayName: Installing Nginx + jobName: Linux_arm64_build + jobDisplayName: "Build: Linux ARM64" + agentOs: Linux + steps: + - script: ./eng/build.sh + --ci + --arch arm64 + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Run build.sh + - script: git clean -xfd src/**/obj/; + ./dockerbuild.sh rhel --ci --nobl --arch arm64 --build-installers --no-build-deps --no-build-nodejs + -p:OnlyPackPlatformSpecificPackages=true -p:BuildRuntimeArchive=false -p:LinuxInstallerType=rpm + -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Build RPM installers + installNodeJs: false artifacts: - - name: MacOS_Test_Logs + - name: Linux_arm64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: MacOS_Test_Results - path: artifacts/TestResults/ + - name: Linux_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_arm64 + + # Build Linux Musl x64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_musl_x64_build + jobDisplayName: "Build: Linux Musl x64" + agentOs: Linux + container: alpine319WithNode + buildArgs: + --arch x64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false + disableComponentGovernance: true + artifacts: + - name: Linux_musl_x64_Logs + path: artifacts/log/ publishOnError: true includeForks: true + - name: Linux_musl_x64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_x64 - - template: jobs/default-build.yml + # Build Linux Musl ARM + - template: .azure/pipelines/jobs/default-build.yml@self parameters: - jobName: Linux_Test - jobDisplayName: "Test: Ubuntu x64" + jobName: Linux_musl_arm_build + jobDisplayName: "Build: Linux Musl ARM" agentOs: Linux - isAzDOTestingJob: true useHostedUbuntu: false - buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) - beforeBuild: - - bash: "./eng/scripts/install-nginx-linux.sh" - displayName: Installing Nginx - - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" - displayName: Increase inotify limit + container: mariner20CrossArmAlpine + buildArgs: + --arch arm + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false artifacts: - - name: Linux_Test_Logs + - name: Linux_musl_arm_Logs path: artifacts/log/ publishOnError: true includeForks: true - - name: Linux_Test_Results - path: artifacts/TestResults/ - publishOnError: true - includeForks: true + - name: Linux_musl_arm_Packages + path: artifacts/packages/ - # Helix x64 - - template: jobs/default-build.yml - parameters: - jobName: Helix_x64 - jobDisplayName: 'Tests: Helix x64' - agentOs: Windows - timeoutInMinutes: 240 - steps: - # Build the shared framework - - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Build shared fx - # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. - - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test - -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true - /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) - displayName: Run build.cmd helix target - env: - HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues - SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_arm + # Build Linux Musl ARM64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_musl_arm64_build + jobDisplayName: "Build: Linux Musl ARM64" + agentOs: Linux + useHostedUbuntu: false + container: mariner20CrossArm64Alpine + buildArgs: + --arch arm64 + --os-name linux-musl + --pack + --all + --no-build-nodejs + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml + $(_BuildArgs) + $(_PublishArgs) + $(_InternalRuntimeDownloadArgs) + installNodeJs: false artifacts: - - name: Helix_logs + - name: Linux_musl_arm64_Logs path: artifacts/log/ publishOnError: true includeForks: true - - # Source build - - template: /eng/common/templates/job/source-build.yml - parameters: - platform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' - skipPublishValidation: true - jobProperties: - timeoutInMinutes: 120 - variables: - # Log environment variables in binary logs to ease debugging - MSBUILDLOGALLENVIRONMENTVARIABLES: true - - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - template: /eng/common/templates/job/source-index-stage1.yml + - name: Linux_musl_arm64_Packages + path: artifacts/packages/ + + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - template: .azure/pipelines/jobs/codesign-xplat.yml@self + parameters: + inputName: Linux_musl_arm64 + + - ${{ if and(ne(parameters.skipTests, 'true'), in(variables['Build.Reason'], 'PullRequest', 'Manual')) }}: + # Test jobs + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Windows_Test + jobDisplayName: "Test: Windows Server x64" + agentOs: Windows + isAzDOTestingJob: true + # Just uploading artifacts/logs/ files can take 15 minutes. Doubling the cancel timeout for this job. + cancelTimeoutInMinutes: 30 + buildArgs: -all -pack -test -binaryLog /p:SkipHelixReadyTests=true /p:SkipIISNewHandlerTests=true /p:SkipIISTests=true + /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:RunBlazorPlaywrightTemplateTests=true + $(_InternalRuntimeDownloadArgs) + beforeBuild: + - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" + displayName: Setup IISExpress test certificates and schema + artifacts: + - name: Windows_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Windows_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: MacOS_Test + jobDisplayName: "Test: macOS" + agentOs: macOS + timeoutInMinutes: 240 + isAzDOTestingJob: true + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-mac.sh" + displayName: Installing Nginx + artifacts: + - name: MacOS_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: MacOS_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Linux_Test + jobDisplayName: "Test: Ubuntu x64" + agentOs: Linux + isAzDOTestingJob: true + useHostedUbuntu: false + buildArgs: --all --test --binaryLog "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs) + beforeBuild: + - bash: "./eng/scripts/install-nginx-linux.sh" + displayName: Installing Nginx + - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" + displayName: Increase inotify limit + artifacts: + - name: Linux_Test_Logs + path: artifacts/log/ + publishOnError: true + includeForks: true + - name: Linux_Test_Results + path: artifacts/TestResults/ + publishOnError: true + includeForks: true + + # Helix x64 + - template: .azure/pipelines/jobs/default-build.yml@self + parameters: + jobName: Helix_x64 + jobDisplayName: 'Tests: Helix x64' + agentOs: Windows + timeoutInMinutes: 240 + steps: + # Build the shared framework + - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Build shared fx + # -noBuildRepoTasks -noBuildNative -noBuild to avoid repeating work done in the previous step. + - script: ./eng/build.cmd -ci -nobl -all -noBuildRepoTasks -noBuildNative -noBuild -test + -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true + /p:CrossgenOutput=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs) + displayName: Run build.cmd helix target + env: + HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + + artifacts: + - name: Helix_logs + path: artifacts/log/ + publishOnError: true + includeForks: true + + # Source build + - template: /eng/common/templates-official/job/source-build.yml@self parameters: - sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true - binlogPath: artifacts/log/Release/Build.binlog - presteps: - - task: NodeTool@0 - displayName: Install Node 18.x - inputs: - versionSpec: 18.x - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - - # Publish to the BAR and perform source indexing. Wait until everything else is done. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml + platform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + buildScript: './eng/build.sh $(_PublishArgs) --no-build-nodejs --no-build-repo-tasks $(_InternalRuntimeDownloadArgs)' + skipPublishValidation: true + jobProperties: + timeoutInMinutes: 120 + variables: + # Log environment variables in binary logs to ease debugging + MSBUILDLOGALLENVIRONMENTVARIABLES: true + + - ${{ if eq(variables.enableSourceIndex, 'true') }}: + - template: /eng/common/templates-official/job/source-index-stage1.yml@self + parameters: + sourceIndexBuildCommand: ./eng/build.cmd -Configuration Release -ci -noBuildJava -binaryLog /p:OnlyPackPlatformSpecificPackages=true + binlogPath: artifacts/log/Release/Build.binlog + presteps: + - task: NodeTool@0 + displayName: Install Node 18.x + inputs: + versionSpec: 18.x + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + + # Publish to the BAR and perform source indexing. Wait until everything else is done. + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + dependsOn: + - Windows_build + - ${{ if ne(variables.PostBuildSign, 'true') }}: + - CodeSign_Xplat_MacOS_arm64 + - CodeSign_Xplat_MacOS_x64 + - CodeSign_Xplat_Linux_x64 + - CodeSign_Xplat_Linux_arm + - CodeSign_Xplat_Linux_arm64 + - CodeSign_Xplat_Linux_musl_x64 + - CodeSign_Xplat_Linux_musl_arm + - CodeSign_Xplat_Linux_musl_arm64 + - ${{ if eq(variables.PostBuildSign, 'true') }}: + - MacOs_arm64_build + - MacOs_x64_build + - Linux_x64_build + - Linux_arm_build + - Linux_arm64_build + - Linux_musl_x64_build + - Linux_musl_arm_build + - Linux_musl_arm64_build + # In addition to the dependencies above that provide assets, ensure the build was successful overall. + - ${{ if in(variables['Build.Reason'], 'Manual') }}: + - Code_check + - ${{ if ne(parameters.skipTests, 'true') }}: + - Windows_Test + - MacOS_Test + - Linux_Test + - Helix_x64 + - ${{ if eq(variables.enableSourceIndex, 'true') }}: + - SourceIndexStage1 + - Source_Build_Managed + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2019 + publishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enablePublishBuildArtifacts: true # publish artifacts/log files + publishAssetsImmediately: true # Don't use a separate stage for darc publishing. + + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self parameters: - dependsOn: - - Windows_build - - ${{ if ne(variables.PostBuildSign, 'true') }}: - - CodeSign_Xplat_MacOS_arm64 - - CodeSign_Xplat_MacOS_x64 - - CodeSign_Xplat_Linux_x64 - - CodeSign_Xplat_Linux_arm - - CodeSign_Xplat_Linux_arm64 - - CodeSign_Xplat_Linux_musl_x64 - - CodeSign_Xplat_Linux_musl_arm - - CodeSign_Xplat_Linux_musl_arm64 - - ${{ if eq(variables.PostBuildSign, 'true') }}: - - MacOs_arm64_build - - MacOs_x64_build - - Linux_x64_build - - Linux_arm_build - - Linux_arm64_build - - Linux_musl_x64_build - - Linux_musl_arm_build - - Linux_musl_arm64_build - # In addition to the dependencies above that provide assets, ensure the build was successful overall. - - ${{ if in(variables['Build.Reason'], 'Manual') }}: - - Code_check - - ${{ if ne(parameters.skipTests, 'true') }}: - - Windows_Test - - MacOS_Test - - Linux_Test - - Helix_x64 - - ${{ if eq(variables.enableSourceIndex, 'true') }}: - - SourceIndexStage1 - - Source_Build_Managed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2019 - publishUsingPipelines: ${{ variables._PublishUsingPipelines }} - enablePublishBuildArtifacts: true # publish artifacts/log files - publishAssetsImmediately: true # Don't use a separate stage for darc publishing. - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(variables.runCodeQL3000, 'true')) }}: - - template: /eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - publishInstallersAndChecksums: true - publishAssetsImmediately: true + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + publishInstallersAndChecksums: true + publishAssetsImmediately: true diff --git a/.azure/pipelines/helix-matrix.yml b/.azure/pipelines/helix-matrix.yml index f28571a50063..b2c9c1a4cc06 100644 --- a/.azure/pipelines/helix-matrix.yml +++ b/.azure/pipelines/helix-matrix.yml @@ -29,7 +29,7 @@ jobs: jobName: Helix_matrix_x64 jobDisplayName: 'Tests: Helix full matrix x64' agentOs: Windows - timeoutInMinutes: 480 + timeoutInMinutes: 300 steps: # Build the shared framework - script: ./eng/build.cmd -ci -nobl -all -pack -arch x64 diff --git a/.azure/pipelines/jobs/codesign-xplat.yml b/.azure/pipelines/jobs/codesign-xplat.yml index c96dc12f93ad..2be4f5b8fc77 100644 --- a/.azure/pipelines/jobs/codesign-xplat.yml +++ b/.azure/pipelines/jobs/codesign-xplat.yml @@ -3,7 +3,7 @@ parameters: inputName: '' jobs: -- template: default-build.yml +- template: default-build.yml@self parameters: codeSign: true dependsOn: @@ -15,7 +15,7 @@ jobs: installNodeJs: false installJdk: false steps: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download ${{ parameters.inputName }} artifacts inputs: artifactName: ${{ parameters.inputName }}_Packages diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 9c9b41d2415e..8a8755e69335 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -79,156 +79,359 @@ parameters: cancelTimeoutInMinutes: 15 jobs: -- template: /eng/common/templates/job/job.yml - parameters: - name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} - displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} - dependsOn: ${{ parameters.dependsOn }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - ${{ if ne(parameters.enableRichCodeNavigation, '') }}: - enableRichCodeNavigation: true - richCodeNavigationLanguage: 'csharp,typescript,java' - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}: - enableMicrobuild: true - enablePublishBuildAssets: true - enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: - enablePublishTestResults: true - testResultsFormat: xUnit # Have no vsTest results in any job. - mergeTestResults: true - testRunTitle: ${{ parameters.testRunTitle }} - enableSbom: ${{ parameters.enableSbom }} - enableTelemetry: true - helixRepo: dotnet/aspnetcore - helixType: build.product/ - workspace: - clean: all - # Map friendly OS names to the right queue - # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md - pool: - ${{ if eq(parameters.agentOs, 'macOS') }}: - vmImage: macOS-11 - ${{ if eq(parameters.agentOs, 'Linux') }}: - ${{ if and(eq(parameters.useHostedUbuntu, true), or(ne(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule'))) }}: - vmImage: ubuntu-20.04 - ${{ if or(eq(parameters.useHostedUbuntu, false), and(eq(variables['System.TeamProject'], 'internal'), notin(variables['Build.Reason'], 'Manual', 'PullRequest', 'Schedule'))) }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: +- ${{ if ne(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates/job/job.yml@self + parameters: + name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} + displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + ${{ if ne(parameters.enableRichCodeNavigation, '') }}: + enableRichCodeNavigation: true + richCodeNavigationLanguage: 'csharp,typescript,java' + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + enablePublishTestResults: true + testResultsFormat: xUnit # Have no vsTest results in any job. + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + enableSbom: ${{ parameters.enableSbom }} + enableTelemetry: true + helixRepo: dotnet/aspnetcore + helixType: build.product/ + workspace: + clean: all + # Map friendly OS names to the right queue + # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md + pool: + ${{ if eq(parameters.agentOs, 'macOS') }}: + vmImage: macOS-11 + ${{ if eq(parameters.agentOs, 'Linux') }}: + ${{ if eq(parameters.useHostedUbuntu, true) }}: + vmImage: ubuntu-20.04 + ${{ if eq(parameters.useHostedUbuntu, false) }}: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 - ${{ if eq(parameters.agentOs, 'Windows') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals 1es-windows-2022-open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + ${{ if ne(parameters.disableComponentGovernance, '') }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + variables: + - AgentOsName: ${{ parameters.agentOs }} + - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping + - BuildScriptArgs: ${{ parameters.buildArgs }} + - _BuildConfig: ${{ parameters.configuration }} + - BuildConfiguration: ${{ parameters.configuration }} + - BuildDirectory: ${{ parameters.buildDirectory }} + - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) + - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + - TeamName: AspNetCore + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 + - ${{ if ne(parameters.codeSign, true) }}: + - _SignType: '' + - LC_ALL: 'en_US.UTF-8' + - LANG: 'en_US.UTF-8' + - LANGUAGE: 'en_US.UTF-8' + # Log environment variables in binary logs to ease debugging + - MSBUILDLOGALLENVIRONMENTVARIABLES: true + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ each variable in parameters.variables }}: + # handle a variable list using "name" and "value" properties + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle name/value pairs (converting them into variable list entries) + # example: + # - [name]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + steps: + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: df -h + displayName: Disk size + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer + displayName: Use XCode 12.5.1 + - checkout: self + clean: true + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/InstallProcDump.ps1 + displayName: Install ProcDump + - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet + displayName: Start background dump collection + - ${{ if eq(parameters.installNodeJs, 'true') }}: + - task: NodeTool@0 + displayName: Install Node 18.x + inputs: + versionSpec: 18.x + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - powershell: ./eng/scripts/InstallJdk.ps1 + displayName: Install JDK 11 + - ${{ if eq(parameters.isAzDOTestingJob, true) }}: + - powershell: | + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" + ./eng/scripts/InstallGoogleChrome.ps1 + displayName: Install Chrome + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" + displayName: Add dotnet tools to path + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" + displayName: Add dotnet tools to path + + - ${{ parameters.beforeBuild }} + + # Add COMPlus_* environment variables to build steps. + - ${{ if ne(parameters.steps, '')}}: + - ${{ each step in parameters.steps }}: + # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. + # Aim here is to avoid having two `env:` properties in the expanded YAML. + - ${{ each pair in step }}: + ${{ if ne(pair.key, 'env') }}: + ${{ pair.key }}: ${{ pair.value }} + env: + # Include the variables we always want. + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + # Expand provided `env:` properties, if any. + ${{ if step.env }}: + ${{ step.env }} + - ${{ if eq(parameters.steps, '')}}: + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + /p:DotNetSignType=$(_SignType) + displayName: Run build.cmd + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.sh + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + + - ${{ parameters.afterBuild }} + + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: df -h && du -h --threshold=50MB .. + displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: df -h && du -h -d 3 .. + displayName: Disk utilization + + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ + displayName: Finish background dump collection + continueOnError: true + condition: always() + + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: eng\scripts\KillProcesses.ps1 + displayName: Kill processes + continueOnError: true + condition: always() + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: eng/scripts/KillProcesses.sh + displayName: Kill processes + continueOnError: true + condition: always() + + - ${{ each artifact in parameters.artifacts }}: + - task: PublishBuildArtifacts@1 + displayName: Upload artifacts from ${{ artifact.path }} + condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) + continueOnError: true + inputs: + # Assume runtime variable values are absolute paths already. + ${{ if startsWith(artifact.path, '$(') }}: + pathToPublish: ${{ artifact.path }} + ${{ if not(startsWith(artifact.path, '$(')) }}: + pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }} + ${{ if eq(artifact.name, '') }}: + artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) + ${{ if ne(artifact.name, '') }}: + artifactName: ${{ artifact.name }} + artifactType: Container + parallel: true + + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ + condition: failed() + displayName: Upload cores + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh + condition: failed() + displayName: Upload cores + + - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + - task: PublishTestResults@2 + displayName: Publish js test results + condition: always() + inputs: + testResultsFormat: JUnit + testResultsFiles: '**/artifacts/log/**/*.junit.xml' + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates-official/job/job.yml@self + parameters: + name: ${{ coalesce(parameters.jobName, parameters.agentOs) }} + displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }} + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + ${{ if ne(parameters.enableRichCodeNavigation, '') }}: + enableRichCodeNavigation: true + richCodeNavigationLanguage: 'csharp,typescript,java' + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}: + enableMicrobuild: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + enablePublishTestResults: true + testResultsFormat: xUnit # Have no vsTest results in any job. + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + enableSbom: ${{ parameters.enableSbom }} + enableTelemetry: true + helixRepo: dotnet/aspnetcore + helixType: build.product/ + workspace: + clean: all + # Map friendly OS names to the right queue + # See https://github.com/dotnet/arcade/blob/master/Documentation/ChoosingAMachinePool.md + pool: + ${{ if eq(parameters.agentOs, 'macOS') }}: + name: Azure Pipelines + image: macOS-11 + os: macOS + ${{ if eq(parameters.agentOs, 'Linux') }}: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2004 + os: linux + ${{ if eq(parameters.agentOs, 'Windows') }}: name: $(DncEngInternalBuildPool) # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing - demands: ImageOverride -equals 1es-windows-2022 - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - ${{ if ne(parameters.disableComponentGovernance, '') }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - variables: - - AgentOsName: ${{ parameters.agentOs }} - - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping - - BuildScriptArgs: ${{ parameters.buildArgs }} - - _BuildConfig: ${{ parameters.configuration }} - - BuildConfiguration: ${{ parameters.configuration }} - - BuildDirectory: ${{ parameters.buildDirectory }} - - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) - - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - - TeamName: AspNetCore - - ${{ if eq(parameters.agentOs, 'Linux') }}: + image: 1es-windows-2022 + os: windows + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + ${{ if ne(parameters.disableComponentGovernance, '') }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + variables: + - AgentOsName: ${{ parameters.agentOs }} + - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping + - BuildScriptArgs: ${{ parameters.buildArgs }} + - _BuildConfig: ${{ parameters.configuration }} + - BuildConfiguration: ${{ parameters.configuration }} + - BuildDirectory: ${{ parameters.buildDirectory }} + - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory) + - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + - TeamName: AspNetCore + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 + - ${{ if eq(parameters.codeSign, true) }}: + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: real + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: test - LC_ALL: 'en_US.UTF-8' - LANG: 'en_US.UTF-8' - LANGUAGE: 'en_US.UTF-8' - - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - - JAVA_HOME: $(Agent.BuildDirectory)\.tools\jdk\win-x64 - - ${{ if or(ne(parameters.codeSign, true), ne(variables['System.TeamProject'], 'internal')) }}: - - _SignType: '' - - ${{ if and(eq(parameters.codeSign, true), eq(variables['System.TeamProject'], 'internal')) }}: - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - _SignType: real - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - _SignType: test - - LC_ALL: 'en_US.UTF-8' - - LANG: 'en_US.UTF-8' - - LANGUAGE: 'en_US.UTF-8' - # Log environment variables in binary logs to ease debugging - - MSBUILDLOGALLENVIRONMENTVARIABLES: true - # Rely on task Arcade injects, not auto-injected build step. - - skipComponentGovernanceDetection: true - - ${{ each variable in parameters.variables }}: - # handle a variable list using "name" and "value" properties - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} + # Log environment variables in binary logs to ease debugging + - MSBUILDLOGALLENVIRONMENTVARIABLES: true + # Rely on task Arcade injects, not auto-injected build step. + - skipComponentGovernanceDetection: true + - ${{ each variable in parameters.variables }}: + # handle a variable list using "name" and "value" properties + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} - # handle name/value pairs (converting them into variable list entries) - # example: - # - [name]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - steps: - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: df -h - displayName: Disk size - - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer - displayName: Use XCode 12.5.1 - - checkout: self - clean: true - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: ./eng/scripts/InstallProcDump.ps1 - displayName: Install ProcDump - - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet - displayName: Start background dump collection - - ${{ if eq(parameters.installNodeJs, 'true') }}: - - task: NodeTool@0 - displayName: Install Node 18.x - inputs: - versionSpec: 18.x - - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: - - powershell: ./eng/scripts/InstallJdk.ps1 - displayName: Install JDK 11 - - ${{ if eq(parameters.isAzDOTestingJob, true) }}: - - powershell: | - Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" - ./eng/scripts/InstallGoogleChrome.ps1 - displayName: Install Chrome - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" - displayName: Add dotnet tools to path - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" - displayName: Add dotnet tools to path + # handle name/value pairs (converting them into variable list entries) + # example: + # - [name]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + steps: + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: df -h + displayName: Disk size + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: sudo xcode-select -s /Applications/Xcode_12.5.1.app/Contents/Developer + displayName: Use XCode 12.5.1 + - checkout: self + clean: true + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/InstallProcDump.ps1 + displayName: Install ProcDump + - powershell: ./eng/scripts/StartDumpCollectionForHangingBuilds.ps1 $(ProcDumpPath)procdump.exe artifacts/dumps/ (Get-Date).AddMinutes(160) dotnet + displayName: Start background dump collection + - ${{ if eq(parameters.installNodeJs, 'true') }}: + - task: NodeTool@0 + displayName: Install Node 18.x + inputs: + versionSpec: 18.x + - ${{ if and(eq(parameters.installJdk, 'true'), eq(parameters.agentOs, 'Windows')) }}: + - powershell: ./eng/scripts/InstallJdk.ps1 + displayName: Install JDK 11 + - ${{ if eq(parameters.isAzDOTestingJob, true) }}: + - powershell: | + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(Build.SourcesDirectory)\artifacts\tmp\selenium\" + ./eng/scripts/InstallGoogleChrome.ps1 + displayName: Install Chrome + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: Write-Host "##vso[task.prependpath]$(DOTNET_CLI_HOME)\.dotnet\tools" + displayName: Add dotnet tools to path + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: echo "##vso[task.prependpath]$(DOTNET_CLI_HOME)/.dotnet/tools" + displayName: Add dotnet tools to path - - ${{ parameters.beforeBuild }} + - ${{ parameters.beforeBuild }} - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if eq(parameters.agentOs, 'Windows') }}: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ if ne(parameters.agentOs, 'Windows') }}: - task: Bash@3 displayName: Setup Private Feeds Credentials @@ -238,100 +441,100 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - # Add COMPlus_* environment variables to build steps. - - ${{ if ne(parameters.steps, '')}}: - - ${{ each step in parameters.steps }}: - # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. - # Aim here is to avoid having two `env:` properties in the expanded YAML. - - ${{ each pair in step }}: - ${{ if ne(pair.key, 'env') }}: - ${{ pair.key }}: ${{ pair.value }} + # Add COMPlus_* environment variables to build steps. + - ${{ if ne(parameters.steps, '')}}: + - ${{ each step in parameters.steps }}: + # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`. + # Aim here is to avoid having two `env:` properties in the expanded YAML. + - ${{ each pair in step }}: + ${{ if ne(pair.key, 'env') }}: + ${{ pair.key }}: ${{ pair.value }} + env: + # Include the variables we always want. + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + # Expand provided `env:` properties, if any. + ${{ if step.env }}: + ${{ step.env }} + - ${{ if eq(parameters.steps, '')}}: + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) + /p:DotNetSignType=$(_SignType) + displayName: Run build.cmd + env: + COMPlus_DbgEnableMiniDump: 1 + COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" + DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) + displayName: Run build.sh env: - # Include the variables we always want. COMPlus_DbgEnableMiniDump: 1 COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - # Expand provided `env:` properties, if any. - ${{ if step.env }}: - ${{ step.env }} - - ${{ if eq(parameters.steps, '')}}: - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) - /p:DotNetSignType=$(_SignType) - displayName: Run build.cmd - env: - COMPlus_DbgEnableMiniDump: 1 - COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" - DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs) - displayName: Run build.sh - env: - COMPlus_DbgEnableMiniDump: 1 - COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core" - DotNetBuildsInternalReadSasToken: $(dotnetbuilds-internal-container-read-token) - - ${{ parameters.afterBuild }} + - ${{ parameters.afterBuild }} - - ${{ if eq(parameters.agentOs, 'Linux') }}: - - script: df -h && du -h --threshold=50MB .. - displayName: Disk utilization - - ${{ if eq(parameters.agentOs, 'macOS') }}: - - script: df -h && du -h -d 3 .. - displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: df -h && du -h --threshold=50MB .. + displayName: Disk utilization + - ${{ if eq(parameters.agentOs, 'macOS') }}: + - script: df -h && du -h -d 3 .. + displayName: Disk utilization - - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: - - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ - displayName: Finish background dump collection - continueOnError: true - condition: always() + - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.isAzDOTestingJob, true)) }}: + - powershell: ./eng/scripts/FinishDumpCollectionForHangingBuilds.ps1 artifacts/dumps/ + displayName: Finish background dump collection + continueOnError: true + condition: always() - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: eng\scripts\KillProcesses.ps1 - displayName: Kill processes - continueOnError: true - condition: always() - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: eng/scripts/KillProcesses.sh - displayName: Kill processes - continueOnError: true - condition: always() + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: eng\scripts\KillProcesses.ps1 + displayName: Kill processes + continueOnError: true + condition: always() + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: eng/scripts/KillProcesses.sh + displayName: Kill processes + continueOnError: true + condition: always() - - ${{ each artifact in parameters.artifacts }}: - - task: PublishBuildArtifacts@1 - displayName: Upload artifacts from ${{ artifact.path }} - condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) - continueOnError: true - inputs: - # Assume runtime variable values are absolute paths already. - ${{ if startsWith(artifact.path, '$(') }}: - pathToPublish: ${{ artifact.path }} - ${{ if not(startsWith(artifact.path, '$(')) }}: - pathToPublish: $(Build.SourcesDirectory)/${{ artifact.path }} - ${{ if eq(artifact.name, '') }}: - artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) - ${{ if ne(artifact.name, '') }}: - artifactName: ${{ artifact.name }} - artifactType: Container - parallel: true + - ${{ each artifact in parameters.artifacts }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: Upload artifacts from ${{ artifact.path }} + condition: and(or(succeeded(), eq('${{ artifact.publishOnError }}', 'true')), or(eq(variables['system.pullrequest.isfork'], false), eq('${{ artifact.includeForks }}', 'true'))) + continueOnError: true + inputs: + # Assume runtime variable values are absolute paths already. + ${{ if startsWith(artifact.path, '$(') }}: + path: ${{ artifact.path }} + ${{ if not(startsWith(artifact.path, '$(')) }}: + path: $(Build.SourcesDirectory)/${{ artifact.path }} + ${{ if eq(artifact.name, '') }}: + artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration) + ${{ if ne(artifact.name, '') }}: + artifactName: ${{ artifact.name }} + artifactType: Container + parallel: true - - ${{ if eq(parameters.agentOs, 'Windows') }}: - - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ - condition: failed() - displayName: Upload cores - - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh - condition: failed() - displayName: Upload cores + - ${{ if eq(parameters.agentOs, 'Windows') }}: + - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/ + condition: failed() + displayName: Upload cores + - ${{ if ne(parameters.agentOs, 'Windows') }}: + - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh + condition: failed() + displayName: Upload cores - - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: - - task: PublishTestResults@2 - displayName: Publish js test results - condition: always() - inputs: - testResultsFormat: JUnit - testResultsFiles: '**/artifacts/log/**/*.junit.xml' - testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js - mergeTestResults: true - buildConfiguration: $(BuildConfiguration) - buildPlatform: $(AgentOsName) + - ${{ if and(eq(parameters.isAzDOTestingJob, true), ne(parameters.enablePublishTestResults, false)) }}: + - task: PublishTestResults@2 + displayName: Publish js test results + condition: always() + inputs: + testResultsFormat: JUnit + testResultsFiles: '**/artifacts/log/**/*.junit.xml' + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-js + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 42da92f402f0..9da9605f0cc3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,7 +10,7 @@ /.vscode/ @captainsafia /.github/ @dotnet/aspnet-build @wtgodbe /.github/*_TEMPLATE/ @dotnet/aspnet-build @wtgodbe @mkArtakMSFT -/.github/workflows/ @dotnet/aspnet-build @wtgodbe @tratcher +/.github/workflows/ @dotnet/aspnet-build @wtgodbe /docs/ @captainsafia @mkArtakMSFT /eng/ @dotnet/aspnet-build @wtgodbe /eng/common/ @dotnet-maestro-bot @@ -21,34 +21,31 @@ /src/Caching/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @captainsafia @halter73 @mgravell /src/Components/ @dotnet/aspnet-blazor-eng /src/Components/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @dotnet/aspnet-blazor-eng -/src/DefaultBuilder/ @tratcher @halter73 -/src/DefaultBuilder/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher +/src/DefaultBuilder/ @halter73 +/src/DefaultBuilder/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review /src/Grpc/ @JamesNK @captainsafia @mgravell /src/Grpc/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @JamesNK @captainsafia @mgravell -/src/Hosting/ @tratcher @halter73 -/src/Hosting/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher -/src/Http/ @tratcher @BrennanConroy @halter73 @captainsafia -/src/Http/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher @BrennanConroy +/src/Hosting/ @halter73 +/src/Hosting/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review +/src/Http/ @BrennanConroy @halter73 @captainsafia +/src/Http/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @BrennanConroy /src/Http/Routing/ @javiercn /src/Http/Routing/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @javiercn /src/HttpClientFactory/ @captainsafia @halter73 /src/HttpClientFactory/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @captainsafia @halter73 /src/Installers/ @dotnet/aspnet-build @wtgodbe /src/JSInterop/ @dotnet/aspnet-blazor-eng -/src/Middleware/ @tratcher @BrennanConroy -/src/Middleware/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher @BrennanConroy +/src/Middleware/ @BrennanConroy +/src/Middleware/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @BrennanConroy /src/Mvc/ @dotnet/minimal-apis /src/Mvc/Mvc.ApiExplorer @captainsafia @halter73 @brunolins16 /src/Mvc/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @dotnet/aspnet-blazor-eng /src/OpenApi @captainsafia @dotnet/minimal-apis /src/ProjectTemplates/Web.ProjectTemplates/content/BlazorServerWeb-CSharp/ @dotnet/aspnet-blazor-eng /src/ProjectTemplates/Web.ProjectTemplates/content/ComponentsWebAssembly-CSharp/ @dotnet/aspnet-blazor-eng -/src/Security/ @tratcher -/src/Security/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher -/src/Servers/ @tratcher @halter73 @BrennanConroy @JamesNK @mgravell -/src/Servers/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @tratcher @halter73 @BrennanConroy @JamesNK @mgravell -/src/Shared/runtime/ @tratcher -/src/Shared/test/Shared.Tests/runtime/ @tratcher +/src/Security/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review +/src/Servers/ @halter73 @BrennanConroy @JamesNK @mgravell +/src/Servers/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @halter73 @BrennanConroy @JamesNK @mgravell /src/SignalR/ @BrennanConroy @halter73 /src/SignalR/**/PublicAPI.*Shipped.txt @dotnet/aspnet-api-review @BrennanConroy @halter73 /src/submodules @dotnet/aspnet-build @wtgodbe diff --git a/.github/workflows/markdownlint-problem-matcher.json b/.github/workflows/markdownlint-problem-matcher.json deleted file mode 100644 index f0741f6b9062..000000000000 --- a/.github/workflows/markdownlint-problem-matcher.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "markdownlint", - "pattern": [ - { - "regexp": "^([^:]*):(\\d+):?(\\d+)?\\s([\\w-\\/]*)\\s(.*)$", - "file": 1, - "line": 2, - "column": 3, - "code": 4, - "message": 5 - } - ] - } - ] -} diff --git a/.github/workflows/markdownlint.yml b/.github/workflows/markdownlint.yml deleted file mode 100644 index e508140dfc95..000000000000 --- a/.github/workflows/markdownlint.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Markdownlint - -permissions: - contents: read - -# run even on changes without markdown changes, so that we can -# make it in GitHub a required check for PR's -on: - pull_request: - -jobs: - lint: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Use Node.js - uses: actions/setup-node@v3 - with: - node-version: 16.x - - name: Run Markdownlint - run: | - echo "::add-matcher::.github/workflows/markdownlint-problem-matcher.json" - npm i -g markdownlint-cli - markdownlint "docs/**/*.md" diff --git a/AspNetCore.sln b/AspNetCore.sln index a41bb737b608..367d27911f8e 100644 --- a/AspNetCore.sln +++ b/AspNetCore.sln @@ -1782,6 +1782,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.AspNetCore.Output EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NotReferencedInWasmCodePackage", "src\Components\test\testassets\NotReferencedInWasmCodePackage\NotReferencedInWasmCodePackage.csproj", "{433F91E4-E39D-4EB0-B798-2998B3969A2C}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Components.WasmRemoteAuthentication", "src\Components\test\testassets\Components.WasmRemoteAuthentication\Components.WasmRemoteAuthentication.csproj", "{8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -10735,6 +10737,22 @@ Global {433F91E4-E39D-4EB0-B798-2998B3969A2C}.Release|x64.Build.0 = Release|Any CPU {433F91E4-E39D-4EB0-B798-2998B3969A2C}.Release|x86.ActiveCfg = Release|Any CPU {433F91E4-E39D-4EB0-B798-2998B3969A2C}.Release|x86.Build.0 = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|arm64.ActiveCfg = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|arm64.Build.0 = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|x64.ActiveCfg = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|x64.Build.0 = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|x86.ActiveCfg = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Debug|x86.Build.0 = Debug|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|Any CPU.Build.0 = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|arm64.ActiveCfg = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|arm64.Build.0 = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|x64.ActiveCfg = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|x64.Build.0 = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|x86.ActiveCfg = Release|Any CPU + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -11615,6 +11633,7 @@ Global {A939893A-B3CD-48F6-80D3-340C8A6E275B} = {AA5ABFBC-177C-421E-B743-005E0FD1248B} {F232B503-D412-45EE-8B31-EFD46B9FA302} = {AA5ABFBC-177C-421E-B743-005E0FD1248B} {433F91E4-E39D-4EB0-B798-2998B3969A2C} = {6126DCE4-9692-4EE2-B240-C65743572995} + {8A021D6D-7935-4AB3-BB47-38D4FF9B0D13} = {6126DCE4-9692-4EE2-B240-C65743572995} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3E8720B3-DBDD-498C-B383-2CC32A054E8F} diff --git a/NuGet.config b/NuGet.config index b118e41dec14..cf1690676806 100644 --- a/NuGet.config +++ b/NuGet.config @@ -6,10 +6,10 @@ - + - + @@ -30,10 +30,10 @@ - + - + diff --git a/eng/Baseline.Designer.props b/eng/Baseline.Designer.props index 537411483eed..49210d5502a1 100644 --- a/eng/Baseline.Designer.props +++ b/eng/Baseline.Designer.props @@ -2,117 +2,117 @@ $(MSBuildAllProjects);$(MSBuildThisFileFullPath) - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 @@ -120,137 +120,137 @@ - 8.0.2 + 8.0.4 - - + + - - + + - - + + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - - - + + + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - + + @@ -258,83 +258,83 @@ - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - + - + - + - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - - - + + + - + - - - + + + - + - - - + + + - + @@ -342,58 +342,58 @@ - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - - + + - - + + - - + + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 @@ -402,7 +402,7 @@ - 8.0.2 + 8.0.4 @@ -410,71 +410,71 @@ - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - + + - - + + - - + + - - + + - 8.0.2 + 8.0.4 - - + + - + - - + + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 - - + + - 8.0.2 + 8.0.4 @@ -490,27 +490,27 @@ - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 @@ -519,79 +519,79 @@ - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - + + - - + + - - + + - 8.0.2 + 8.0.4 - - + + - - + + - - + + - - + + @@ -599,190 +599,193 @@ - 8.0.2 + 8.0.4 - + - + - + - + - + - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - - - - + + + + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + + - + + - + + - 8.0.2 + 8.0.4 - + - + - + - + - + - + - 8.0.2 + 8.0.4 @@ -798,48 +801,48 @@ - 8.0.2 + 8.0.4 - + - + - + - + - + - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - - - + + + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 @@ -849,7 +852,7 @@ - 8.0.2 + 8.0.4 @@ -858,79 +861,79 @@ - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - - - + + + - - - + + + - - - + + + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + @@ -939,7 +942,7 @@ - + @@ -947,46 +950,46 @@ - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - + - + - 8.0.2 + 8.0.4 - 8.0.2 + 8.0.4 - + - 8.0.2 + 8.0.4 diff --git a/eng/Baseline.xml b/eng/Baseline.xml index b2335226206e..110083387ae6 100644 --- a/eng/Baseline.xml +++ b/eng/Baseline.xml @@ -4,110 +4,110 @@ This file contains a list of all the packages and their versions which were rele Update this list when preparing for a new patch. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/Publishing.props b/eng/Publishing.props index 943018d2a694..8765a8714c20 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,6 +1,7 @@ 3 + true diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8df7c9d6944c..1eb8cf758972 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -9,37 +9,37 @@ --> - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - c2f996919858eb11d2836705d47b531b5a174c79 + 6a2be34d045329d9eff9536ec824226696d53e00 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -121,9 +121,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -185,13 +185,13 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://github.com/dotnet/source-build-externals - 83274d94c7e2ff21081b0d75ecbec2da2241f831 + 300e99190e6ae1983681694dbdd5f75f0c692081 @@ -203,9 +203,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + 2d7eea252964e69be94cb9c847b371b23e4dd470 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -239,9 +239,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 5535e31a712343a63f5d7d796cd874e563e5ac14 + 3b8b000a0e115700b18265d8ec8c6307056dc94d https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -275,17 +275,17 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime 5535e31a712343a63f5d7d796cd874e563e5ac14 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d https://dev.azure.com/dnceng/internal/_git/dotnet-runtime @@ -316,31 +316,31 @@ Win-x64 is used here because we have picked an arbitrary runtime identifier to flow the version of the latest NETCore.App runtime. All Runtime.$rid packages should have the same version. --> - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d https://github.com/dotnet/xdt 9a1c3e1b7f0c8763d4c96e593961a61a72679a7b - + https://github.com/dotnet/source-build-reference-packages - 453a37ef7ae6c335cd49b3b9ab7713c87faeb265 + 79827eed138fd2575a8b24820b4f385ee4ffb6e6 @@ -368,34 +368,34 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 9f4b1f5d664afdfc80e1508ab7ed099dff210fbd + 3b8b000a0e115700b18265d8ec8c6307056dc94d https://github.com/dotnet/winforms abda8e3bfa78319363526b5a5f86863ec979940e - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac https://github.com/dotnet/extensions diff --git a/eng/Versions.props b/eng/Versions.props index eec156015f1a..16ba86153b2e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,10 +8,10 @@ 8 0 - 3 + 6 - true + false 7.1.2 8.0.0 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3-servicing.24114.23 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6-servicing.24267.15 8.0.0 8.0.0 8.0.0 @@ -92,7 +92,7 @@ 8.0.0 8.0.0 8.0.0 - 8.0.3-servicing.24114.23 + 8.0.6-servicing.24267.15 8.0.0 8.0.0 8.0.0 @@ -108,9 +108,9 @@ 8.0.0 8.0.2 8.0.0 - 8.0.3-servicing.24114.23 + 8.0.6-servicing.24267.15 8.0.0 - 8.0.0 + 8.0.1 8.0.0 8.0.0 8.0.0-rtm.23520.14 @@ -120,7 +120,7 @@ 8.0.0 8.0.0 8.0.0 - 8.0.0 + 8.0.1 8.0.0 8.0.0 8.0.0 @@ -128,9 +128,9 @@ 8.0.0 8.0.0 8.0.0 - 8.0.3-servicing.24114.23 + 8.0.6-servicing.24267.15 - 8.0.3-servicing.24114.23 + 8.0.6-servicing.24267.15 8.0.0 8.0.1 @@ -142,14 +142,14 @@ 8.1.0-preview.23604.1 8.1.0-preview.23604.1 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 + 8.0.6 4.8.0-3.23518.7 4.8.0-3.23518.7 @@ -161,13 +161,13 @@ 6.2.4 6.2.4 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 - 8.0.0-alpha.1.24065.1 + 8.0.0-alpha.1.24175.3 - 8.0.0-alpha.1.24061.1 + 8.0.0-alpha.1.24163.3 2.0.0-beta-23228-03 @@ -310,7 +310,7 @@ 1.0.2 13.0.3 13.0.4 - 2.4.0 + 2.5.2 1.28.0 3.0.0 7.2.4 @@ -318,7 +318,7 @@ 4.17.0 1.4.0 4.0.0 - 2.6.122 + 2.7.27 5.0.0 6.4.0 2.0.3 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925f2..efa2fd72bfaa 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index f5c1ec7eafeb..2d5660642b8d 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 000000000000..1f035fee73f4 --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,264 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 000000000000..52b4d05d3f8d --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 000000000000..589ac80a18b7 --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,155 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 000000000000..f193dfbe2366 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2 + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 000000000000..43ee0c202fc7 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,85 @@ +parameters: + runAsPublic: false + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - template: /eng/common/templates/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 000000000000..b68d3c2f3199 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 000000000000..857a0f8ba43e --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 000000000000..08e5db9bb116 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 000000000000..c24193acfc98 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 000000000000..da1f40958b45 --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..0c87f149a4ad --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 000000000000..da669030daf6 --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 000000000000..f67a210d62f3 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 000000000000..eba58109b52c --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 000000000000..cbba0596709d --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 000000000000..9b4a5ffa30a7 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 000000000000..07426fde05d8 --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 000000000000..1bf43bf807af --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 000000000000..04012fed182a --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 000000000000..83d97a26a01f --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 000000000000..3eb7e2d5f840 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 000000000000..829f17c34d11 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 000000000000..1f308b24efc4 --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022 + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 000000000000..dbdd66d4a4b3 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f98..8ec5c4f2d9f9 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02d8..43ee0c202fc7 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -40,16 +41,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -61,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c91..cbba0596709d 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa5..2b21eae42732 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/eng/targets/Helix.Common.props b/eng/targets/Helix.Common.props index e586fa5efc16..426386e98ebf 100644 --- a/eng/targets/Helix.Common.props +++ b/eng/targets/Helix.Common.props @@ -1,12 +1,12 @@ - (AlmaLinux.8.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-helix-amd64 - (Alpine.316.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-amd64 - (Debian.11.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - (Fedora.34.Amd64.Open)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - (Mariner)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix - (Debian.12.Arm64.Open)ubuntu.2004.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm64v8 + (AlmaLinux.8.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-helix-amd64 + (Alpine.317.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-amd64 + (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 + (Fedora.38.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38-helix + (Mariner)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix + (Debian.12.Arm64.Open)ubuntu.2204.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm64v8 false @@ -42,9 +42,9 @@ - + - + diff --git a/eng/targets/Wix.Common.props b/eng/targets/Wix.Common.props index 1be1ce45a3e6..d8fcb93ea39a 100644 --- a/eng/targets/Wix.Common.props +++ b/eng/targets/Wix.Common.props @@ -4,7 +4,7 @@ 2.0 3.14 - 3.14.0-8606.20240208.1 + $(MicrosoftSignedWixVersion) diff --git a/eng/tools/RepoTasks/RepoTasks.csproj b/eng/tools/RepoTasks/RepoTasks.csproj index aa07cbc9dc51..a15b19e51dcb 100644 --- a/eng/tools/RepoTasks/RepoTasks.csproj +++ b/eng/tools/RepoTasks/RepoTasks.csproj @@ -34,7 +34,7 @@ - + diff --git a/global.json b/global.json index cf8518ed1b74..56487e3d9c92 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "sdk": { - "version": "8.0.102" + "version": "8.0.104" }, "tools": { - "dotnet": "8.0.102", + "dotnet": "8.0.104", "runtimes": { "dotnet/x86": [ "$(MicrosoftNETCoreBrowserDebugHostTransportVersion)" @@ -27,7 +27,7 @@ }, "msbuild-sdks": { "Yarn.MSBuild": "1.22.19", - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24113.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24266.3" } } diff --git a/src/Caching/StackExchangeRedis/src/RedisCache.Log.cs b/src/Caching/StackExchangeRedis/src/RedisCache.Log.cs index c20e93f52cef..5569613a2eb7 100644 --- a/src/Caching/StackExchangeRedis/src/RedisCache.Log.cs +++ b/src/Caching/StackExchangeRedis/src/RedisCache.Log.cs @@ -12,5 +12,8 @@ private static partial class Log { [LoggerMessage(1, LogLevel.Warning, "Could not determine the Redis server version. Falling back to use HMSET command instead of HSET.", EventName = "CouldNotDetermineServerVersion")] public static partial void CouldNotDetermineServerVersion(ILogger logger, Exception exception); + + [LoggerMessage(2, LogLevel.Debug, "Unable to add library name suffix.", EventName = "UnableToAddLibraryNameSuffix")] + internal static partial void UnableToAddLibraryNameSuffix(ILogger logger, Exception exception); } } diff --git a/src/Caching/StackExchangeRedis/src/RedisCache.cs b/src/Caching/StackExchangeRedis/src/RedisCache.cs index 9a896e0e5a7c..b346bbd24b4d 100644 --- a/src/Caching/StackExchangeRedis/src/RedisCache.cs +++ b/src/Caching/StackExchangeRedis/src/RedisCache.cs @@ -257,14 +257,7 @@ private IDatabase Connect() IConnectionMultiplexer connection; if (_options.ConnectionMultiplexerFactory is null) { - if (_options.ConfigurationOptions is not null) - { - connection = ConnectionMultiplexer.Connect(_options.ConfigurationOptions); - } - else - { - connection = ConnectionMultiplexer.Connect(_options.Configuration!); - } + connection = ConnectionMultiplexer.Connect(_options.GetConfiguredOptions()); } else { @@ -308,7 +301,7 @@ private async ValueTask ConnectSlowAsync(CancellationToken token) IConnectionMultiplexer connection; if (_options.ConnectionMultiplexerFactory is null) { - connection = await ConnectionMultiplexer.ConnectAsync(_options.GetConfiguredOptions("asp.net DC")).ConfigureAwait(false); + connection = await ConnectionMultiplexer.ConnectAsync(_options.GetConfiguredOptions()).ConfigureAwait(false); } else { @@ -332,6 +325,7 @@ private void PrepareConnection(IConnectionMultiplexer connection) WriteTimeTicks(ref _lastConnectTicks, DateTimeOffset.UtcNow); ValidateServerFeatures(connection); TryRegisterProfiler(connection); + TryAddSuffix(connection); } private void ValidateServerFeatures(IConnectionMultiplexer connection) @@ -369,6 +363,19 @@ private void TryRegisterProfiler(IConnectionMultiplexer connection) } } + private void TryAddSuffix(IConnectionMultiplexer connection) + { + try + { + connection.AddLibraryNameSuffix("aspnet"); + connection.AddLibraryNameSuffix("DC"); + } + catch (Exception ex) + { + Log.UnableToAddLibraryNameSuffix(_logger, ex); + } + } + private byte[]? GetAndRefresh(string key, bool getData) { ArgumentNullThrowHelper.ThrowIfNull(key); diff --git a/src/Caching/StackExchangeRedis/src/RedisCacheOptions.cs b/src/Caching/StackExchangeRedis/src/RedisCacheOptions.cs index f6386dc7b00a..23aad4f9e642 100644 --- a/src/Caching/StackExchangeRedis/src/RedisCacheOptions.cs +++ b/src/Caching/StackExchangeRedis/src/RedisCacheOptions.cs @@ -59,18 +59,13 @@ static bool GetDefaultValue() => set => _useForceReconnect = value; } - internal ConfigurationOptions GetConfiguredOptions(string libSuffix) + internal ConfigurationOptions GetConfiguredOptions() { - var options = ConfigurationOptions?.Clone() ?? ConfigurationOptions.Parse(Configuration!); + var options = ConfigurationOptions ?? ConfigurationOptions.Parse(Configuration!); // we don't want an initially unavailable server to prevent DI creating the service itself options.AbortOnConnectFail = false; - if (!string.IsNullOrWhiteSpace(libSuffix)) - { - var provider = DefaultOptionsProvider.GetProvider(options.EndPoints); - options.LibraryName = $"{provider.LibraryName} {libSuffix}"; - } return options; } } diff --git a/src/Components/Components.slnf b/src/Components/Components.slnf index 4ea1556f741f..a35d7e8e8686 100644 --- a/src/Components/Components.slnf +++ b/src/Components/Components.slnf @@ -52,6 +52,7 @@ "src\\Components\\test\\testassets\\BasicTestApp\\BasicTestApp.csproj", "src\\Components\\test\\testassets\\Components.TestServer\\Components.TestServer.csproj", "src\\Components\\test\\testassets\\Components.WasmMinimal\\Components.WasmMinimal.csproj", + "src\\Components\\test\\testassets\\Components.WasmRemoteAuthentication\\Components.WasmRemoteAuthentication.csproj", "src\\Components\\test\\testassets\\ComponentsApp.App\\ComponentsApp.App.csproj", "src\\Components\\test\\testassets\\ComponentsApp.Server\\ComponentsApp.Server.csproj", "src\\Components\\test\\testassets\\GlobalizationWasmApp\\GlobalizationWasmApp.csproj", diff --git a/src/Components/Endpoints/src/RazorComponentEndpointInvoker.cs b/src/Components/Endpoints/src/RazorComponentEndpointInvoker.cs index 41efc89f9749..02bf4e3a3467 100644 --- a/src/Components/Endpoints/src/RazorComponentEndpointInvoker.cs +++ b/src/Components/Endpoints/src/RazorComponentEndpointInvoker.cs @@ -9,6 +9,7 @@ using Microsoft.AspNetCore.Components.Endpoints.Rendering; using Microsoft.AspNetCore.Diagnostics; using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.WebUtilities; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -116,6 +117,16 @@ await EndpointHtmlRenderer.InitializeStandardComponentServicesAsync( } } + if (!quiesceTask.IsCompleted) + { + // An incomplete QuiescenceTask indicates there may be streaming rendering updates. + // Disable all response buffering and compression on IIS like SignalR's ServerSentEventsServerTransport does. + var bufferingFeature = context.Features.GetRequiredFeature(); + bufferingFeature.DisableBuffering(); + + context.Response.Headers.ContentEncoding = "identity"; + } + // Importantly, we must not yield this thread (which holds exclusive access to the renderer sync context) // in between the first call to htmlContent.WriteTo and the point where we start listening for subsequent // streaming SSR batches (inside SendStreamingUpdatesAsync). Otherwise some other code might dispatch to the diff --git a/src/Components/WebAssembly/WebAssembly.Authentication/src/Services/RemoteAuthenticationService.cs b/src/Components/WebAssembly/WebAssembly.Authentication/src/Services/RemoteAuthenticationService.cs index 62d9fa5265e4..12d315bdaf60 100644 --- a/src/Components/WebAssembly/WebAssembly.Authentication/src/Services/RemoteAuthenticationService.cs +++ b/src/Components/WebAssembly/WebAssembly.Authentication/src/Services/RemoteAuthenticationService.cs @@ -108,7 +108,7 @@ public virtual async Task RemoteAuthenticationContext context) { await EnsureAuthService(); - var result = await JsRuntime.InvokeAsync>("AuthenticationService.signIn", context); + var result = await JSInvokeWithContextAsync, RemoteAuthenticationResult>("AuthenticationService.signIn", context); await UpdateUserOnSuccess(result); return result; @@ -130,7 +130,7 @@ public virtual async Task RemoteAuthenticationContext context) { await EnsureAuthService(); - var result = await JsRuntime.InvokeAsync>("AuthenticationService.signOut", context); + var result = await JSInvokeWithContextAsync, RemoteAuthenticationResult>("AuthenticationService.signOut", context); await UpdateUserOnSuccess(result); return result; @@ -187,6 +187,11 @@ public virtual async ValueTask RequestAccessToken(AccessToken } : null); } + // JSRuntime.InvokeAsync does not properly annotate all arguments with DynamicallyAccessedMembersAttribute. https://github.com/dotnet/aspnetcore/issues/39839 + // Calling JsRuntime.InvokeAsync directly results allows the RemoteAuthenticationContext.State getter to be trimmed. https://github.com/dotnet/aspnetcore/issues/49956 + private ValueTask JSInvokeWithContextAsync<[DynamicallyAccessedMembers(JsonSerialized)] TContext, [DynamicallyAccessedMembers(JsonSerialized)] TResult>( + string identifier, TContext context) => JsRuntime.InvokeAsync(identifier, context); + private string GetReturnUrl(string? customReturnUrl) => customReturnUrl != null ? Navigation.ToAbsoluteUri(customReturnUrl).AbsoluteUri : Navigation.Uri; diff --git a/src/Components/WebView/test/E2ETest/BasicBlazorHybridTest.cs b/src/Components/WebView/test/E2ETest/BasicBlazorHybridTest.cs index df64496c253c..9cd357a02f5d 100644 --- a/src/Components/WebView/test/E2ETest/BasicBlazorHybridTest.cs +++ b/src/Components/WebView/test/E2ETest/BasicBlazorHybridTest.cs @@ -18,21 +18,21 @@ public void Run() // Note: This test produces *a lot* of debug output to aid when debugging failures. The only output // that is necessary for the functioning of this test is the "Test passed" at the end of this method. - Console.WriteLine($"Current directory: {Environment.CurrentDirectory}"); - Console.WriteLine($"Current assembly: {typeof(Program).Assembly.Location}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Current directory: {Environment.CurrentDirectory}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Current assembly: {typeof(Program).Assembly.Location}"); var thisProgramDir = Path.GetDirectoryName(typeof(Program).Assembly.Location); // Add correct runtime sub-folder to PATH to ensure native files are discovered (this is supposed to happen automatically, but somehow it doesn't...) var newNativePath = Path.Combine(thisProgramDir, "runtimes", RuntimeInformation.RuntimeIdentifier, "native"); - Console.WriteLine($"Adding new native path: {newNativePath}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Adding new native path: {newNativePath}"); Environment.SetEnvironmentVariable("PATH", newNativePath + ";" + Environment.GetEnvironmentVariable("PATH")); - Console.WriteLine($"New PATH env var: {Environment.GetEnvironmentVariable("PATH")}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] New PATH env var: {Environment.GetEnvironmentVariable("PATH")}"); var thisAppFiles = Directory.GetFiles(thisProgramDir, "*", SearchOption.AllDirectories).ToArray(); - Console.WriteLine($"Found {thisAppFiles.Length} files in this app:"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Found {thisAppFiles.Length} files in this app:"); foreach (var file in thisAppFiles) { - Console.WriteLine($"\t{file}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] \t{file}"); } var hostPage = "wwwroot/webviewtesthost.html"; @@ -41,7 +41,7 @@ public void Run() serviceCollection.AddBlazorWebView(); serviceCollection.AddSingleton(); - Console.WriteLine($"Creating BlazorWindow..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Creating BlazorWindow..."); BlazorWindow mainWindow = null; try { @@ -53,11 +53,11 @@ public void Run() } catch (Exception ex) { - Console.WriteLine($"Exception {ex.GetType().FullName} while creating window: {ex.Message}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Exception {ex.GetType().FullName} while creating window: {ex.Message}"); Console.WriteLine(ex.StackTrace); } - Console.WriteLine($"Hooking exception handler..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Hooking exception handler..."); AppDomain.CurrentDomain.UnhandledException += (sender, error) => { Console.Write( @@ -65,15 +65,15 @@ public void Run() error.ExceptionObject.ToString() + Environment.NewLine); }; - Console.WriteLine($"Setting up root components..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Setting up root components..."); mainWindow.RootComponents.Add("root"); - Console.WriteLine($"Running window..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Running window..."); const string NewControlDivValueMessage = "wvt:NewControlDivValue"; var isWebViewReady = false; - Console.WriteLine($"RegisterWebMessageReceivedHandler..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] RegisterWebMessageReceivedHandler..."); mainWindow.PhotinoWindow.RegisterWebMessageReceivedHandler((s, msg) => { if (!msg.StartsWith("__bwv:", StringComparison.Ordinal)) @@ -90,29 +90,33 @@ public void Run() }); var testPassed = false; + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Attaching WindowCreated handler..."); mainWindow.PhotinoWindow.WindowCreated += (s, e) => { + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] In WindowCreated event..."); Task.Run(async () => { + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] In async test task..."); + try { // This is the actual test logic here (wait for WebView, click button, verify updates, etc.) // 1. Wait for WebView ready - Console.WriteLine($"Waiting for WebView ready..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Waiting for WebView ready..."); var isWebViewReadyRetriesLeft = 20; while (!isWebViewReady) { - Console.WriteLine($"WebView not ready yet, waiting 1sec..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] WebView not ready yet, waiting 1sec..."); await Task.Delay(1000); isWebViewReadyRetriesLeft--; if (isWebViewReadyRetriesLeft == 0) { - Console.WriteLine($"WebView never became ready, failing the test..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] WebView never became ready, failing the test..."); return; } } - Console.WriteLine($"WebView is ready!"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] WebView is ready!"); // 2. Check TestPage starting state if (!await WaitForControlDiv(mainWindow.PhotinoWindow, controlValueToWaitFor: "0")) @@ -130,7 +134,7 @@ public void Run() } // 5. If we get here, it all worked! - Console.WriteLine($"All tests passed!"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] All tests passed!"); testPassed = true; } catch (Exception ex) @@ -153,16 +157,16 @@ public void Run() } catch (Exception ex) { - Console.WriteLine($"Exception while running window: {ex}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Exception while running window: {ex}"); } // This line is what's required for the test to be considered as passed. The xUnit test in WebViewManagerE2ETests checks // that this reports success and that decides if the test is pass/fail. - Console.WriteLine($"Test passed? {testPassed}"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Test passed? {testPassed}"); } const int MaxWaitTimes = 30; - const int WaitTimeInMS = 250; + const int WaitTimeInMS = 1000; public async Task WaitForControlDiv(PhotinoWindow photinoWindow, string controlValueToWaitFor) { @@ -172,20 +176,20 @@ public async Task WaitForControlDiv(PhotinoWindow photinoWindow, string co // Tell WebView to report the current controlDiv value (this is inside the loop because // it's possible for this to execute before the WebView has finished processing previous // C#-generated events, such as WebView button clicks). - Console.WriteLine($"Asking WebView for current controlDiv value..."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Asking WebView for current controlDiv value..."); photinoWindow.SendWebMessage($"wvt:GetControlDivValue"); // And wait for the value to appear if (_latestControlDivValue == controlValueToWaitFor) { - Console.WriteLine($"WebView reported the expected controlDiv value of {controlValueToWaitFor}!"); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] WebView reported the expected controlDiv value of {controlValueToWaitFor}!"); return true; } - Console.WriteLine($"Waiting for controlDiv to have value '{controlValueToWaitFor}', but it's still '{_latestControlDivValue}', so waiting {WaitTimeInMS}ms."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Waiting for controlDiv to have value '{controlValueToWaitFor}', but it's still '{_latestControlDivValue}', so waiting {WaitTimeInMS}ms."); await Task.Delay(WaitTimeInMS); } - Console.WriteLine($"Waited {MaxWaitTimes * WaitTimeInMS}ms but couldn't get controlDiv to have value '{controlValueToWaitFor}' (last value is '{_latestControlDivValue}')."); + Console.WriteLine($"[{DateTime.Now.ToLongTimeString()}] Waited {MaxWaitTimes * WaitTimeInMS}ms but couldn't get controlDiv to have value '{controlValueToWaitFor}' (last value is '{_latestControlDivValue}')."); return false; } } diff --git a/src/Components/WebView/test/E2ETest/WebViewManagerE2ETests.cs b/src/Components/WebView/test/E2ETest/WebViewManagerE2ETests.cs index 228cd7b0a96b..0f944bb5795d 100644 --- a/src/Components/WebView/test/E2ETest/WebViewManagerE2ETests.cs +++ b/src/Components/WebView/test/E2ETest/WebViewManagerE2ETests.cs @@ -3,10 +3,11 @@ using System.Diagnostics; using Microsoft.AspNetCore.Testing; +using Xunit.Abstractions; namespace Microsoft.AspNetCore.Components.WebViewE2E.Test; -public class WebViewManagerE2ETests +public class WebViewManagerE2ETests(ITestOutputHelper output) { // Skips: // - Ubuntu is skipped due to this error: @@ -39,7 +40,9 @@ public async Task CanLaunchPhotinoWebViewAndClickButton() var testProgramOutput = photinoProcess.StandardOutput.ReadToEnd(); - await photinoProcess.WaitForExitAsync().TimeoutAfter(TimeSpan.FromSeconds(30)); + await photinoProcess.WaitForExitAsync().TimeoutAfter(TimeSpan.FromSeconds(60)); + + output.WriteLine(testProgramOutput); // The test app reports its own results by calling Console.WriteLine(), so here we only need to verify that // the test internally believes it passed (and we trust it!). diff --git a/src/Components/test/E2ETest/Infrastructure/ServerFixtures/AspNetSiteServerFixture.cs b/src/Components/test/E2ETest/Infrastructure/ServerFixtures/AspNetSiteServerFixture.cs index 5bd6846cf3b9..955061b702ab 100644 --- a/src/Components/test/E2ETest/Infrastructure/ServerFixtures/AspNetSiteServerFixture.cs +++ b/src/Components/test/E2ETest/Infrastructure/ServerFixtures/AspNetSiteServerFixture.cs @@ -32,7 +32,7 @@ protected override IHost CreateWebHost() } var assembly = ApplicationAssembly ?? BuildWebHostMethod.Method.DeclaringType.Assembly; - var sampleSitePath = DefaultGetContentRoot(assembly); + var sampleSitePath = GetContentRootMethod(assembly); var host = "127.0.0.1"; if (E2ETestOptions.Instance.SauceTest) diff --git a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj index 763b968c63b1..b8ad7c0303a4 100644 --- a/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj +++ b/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj @@ -1,4 +1,4 @@ - + diff --git a/src/Components/test/E2ETest/ServerRenderingTests/FormHandlingTests/FormWithParentBindingContextTest.cs b/src/Components/test/E2ETest/ServerRenderingTests/FormHandlingTests/FormWithParentBindingContextTest.cs index 271d6bf686e5..80423bd8e8d7 100644 --- a/src/Components/test/E2ETest/ServerRenderingTests/FormHandlingTests/FormWithParentBindingContextTest.cs +++ b/src/Components/test/E2ETest/ServerRenderingTests/FormHandlingTests/FormWithParentBindingContextTest.cs @@ -1466,7 +1466,7 @@ public void SubmitButtonFormenctypeAttributeOverridesEnhancedFormEnctype() [Fact] public void EnhancedFormThatCallsNavigationManagerRefreshDoesNotPushHistoryEntry() { - GoTo("about:blank"); + Navigate("about:blank"); var startUrl = Browser.Url; GoTo("forms/form-that-calls-navigation-manager-refresh"); @@ -1488,7 +1488,7 @@ public void EnhancedFormThatCallsNavigationManagerRefreshDoesNotPushHistoryEntry [Fact] public void EnhancedFormThatCallsNavigationManagerRefreshDoesNotPushHistoryEntry_Streaming() { - GoTo("about:blank"); + Navigate("about:blank"); var startUrl = Browser.Url; GoTo("forms/form-that-calls-navigation-manager-refresh-streaming"); diff --git a/src/Components/test/E2ETest/ServerRenderingTests/StreamingRenderingTest.cs b/src/Components/test/E2ETest/ServerRenderingTests/StreamingRenderingTest.cs index ddd8ff262230..76ba4abec238 100644 --- a/src/Components/test/E2ETest/ServerRenderingTests/StreamingRenderingTest.cs +++ b/src/Components/test/E2ETest/ServerRenderingTests/StreamingRenderingTest.cs @@ -3,13 +3,13 @@ using System.Globalization; using System.Net.Http; -using System.Net.Http.Headers; using System.Text; using System.Text.RegularExpressions; using Components.TestServer.RazorComponents; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure; using Microsoft.AspNetCore.Components.E2ETest.Infrastructure.ServerFixtures; using Microsoft.AspNetCore.E2ETesting; +using Microsoft.Net.Http.Headers; using OpenQA.Selenium; using TestServer; using Xunit.Abstractions; @@ -30,13 +30,29 @@ public override Task InitializeAsync() => InitializeAsync(BrowserFixture.StreamingContext); [Fact] - public void CanRenderNonstreamingPageWithoutInjectingStreamingMarkers() + public async Task CanRenderNonstreamingPageWithoutInjectingStreamingMarkersOrHeaders() { Navigate(ServerPathBase); Browser.Equal("Hello", () => Browser.Exists(By.TagName("h1")).Text); Assert.DoesNotContain("> +{ + public readonly bool TestTrimmedApps = typeof(ToggleExecutionModeServerFixture<>).Assembly + .GetCustomAttributes() + .First(m => m.Key == "Microsoft.AspNetCore.E2ETesting.TestTrimmedOrMultithreadingApps") + .Value == "true"; + + public RemoteAuthenticationTest( + BrowserFixture browserFixture, + BasicTestAppServerSiteFixture serverFixture, + ITestOutputHelper output) + : base(browserFixture, serverFixture, output) + { + serverFixture.ApplicationAssembly = typeof(RemoteAuthenticationStartup).Assembly; + + if (TestTrimmedApps) + { + serverFixture.BuildWebHostMethod = BuildPublishedWebHost; + serverFixture.GetContentRootMethod = GetPublishedContentRoot; + } + } + + [Fact] + public void NavigateToLogin_PreservesExtraQueryParams() + { + // If the preservedExtraQueryParams passed to NavigateToLogin by RedirectToLogin gets trimmed, + // the OIDC endpoints will fail to authenticate the user. + Navigate("/subdir/test-remote-authentication"); + + var heading = Browser.Exists(By.TagName("h1")); + Browser.Equal("Hello, Jane Doe!", () => heading.Text); + } + + private static IHost BuildPublishedWebHost(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureLogging((ctx, lb) => + { + TestSink sink = new TestSink(); + lb.AddProvider(new TestLoggerProvider(sink)); + lb.Services.AddSingleton(sink); + }) + .ConfigureWebHostDefaults(webHostBuilder => + { + webHostBuilder.UseStartup(); + // Avoid UseStaticAssets or we won't use the trimmed published output. + }) + .Build(); + + private static string GetPublishedContentRoot(Assembly assembly) + { + var contentRoot = Path.Combine(AppContext.BaseDirectory, "trimmed-or-threading", assembly.GetName().Name); + + if (!Directory.Exists(contentRoot)) + { + throw new DirectoryNotFoundException($"Test is configured to use trimmed outputs, but trimmed outputs were not found in {contentRoot}."); + } + + return contentRoot; + } +} diff --git a/src/Components/test/E2ETest/Tests/WebAssemblyPrerenderedTest.cs b/src/Components/test/E2ETest/Tests/WebAssemblyPrerenderedTest.cs index 9007e18ab483..1b108484b9db 100644 --- a/src/Components/test/E2ETest/Tests/WebAssemblyPrerenderedTest.cs +++ b/src/Components/test/E2ETest/Tests/WebAssemblyPrerenderedTest.cs @@ -56,7 +56,7 @@ private void WaitUntilLoaded() private static string GetPublishedContentRoot(Assembly assembly) { - var contentRoot = Path.Combine(AppContext.BaseDirectory, "trimmed", assembly.GetName().Name); + var contentRoot = Path.Combine(AppContext.BaseDirectory, "trimmed-or-threading", assembly.GetName().Name); if (!Directory.Exists(contentRoot)) { diff --git a/src/Components/test/testassets/Components.TestServer/Components.TestServer.csproj b/src/Components/test/testassets/Components.TestServer/Components.TestServer.csproj index 6279d994bc7b..16d1475fc03c 100644 --- a/src/Components/test/testassets/Components.TestServer/Components.TestServer.csproj +++ b/src/Components/test/testassets/Components.TestServer/Components.TestServer.csproj @@ -25,11 +25,13 @@ + + diff --git a/src/Components/test/testassets/Components.TestServer/Program.cs b/src/Components/test/testassets/Components.TestServer/Program.cs index 1c6156c605d0..a3206465989d 100644 --- a/src/Components/test/testassets/Components.TestServer/Program.cs +++ b/src/Components/test/testassets/Components.TestServer/Program.cs @@ -19,6 +19,7 @@ public static async Task Main(string[] args) var createIndividualHosts = new Dictionary { ["Client authentication"] = (BuildWebHost(CreateAdditionalArgs(args)), "/subdir"), + ["Remote client authentication"] = (BuildWebHost(CreateAdditionalArgs(args)), "/subdir"), ["Server authentication"] = (BuildWebHost(CreateAdditionalArgs(args)), "/subdir"), ["CORS (WASM)"] = (BuildWebHost(CreateAdditionalArgs(args)), "/subdir"), ["Prerendering (Server-side)"] = (BuildWebHost(CreateAdditionalArgs(args)), "/prerendered"), diff --git a/src/Components/test/testassets/Components.TestServer/RazorComponents/RemoteAuthenticationApp.razor b/src/Components/test/testassets/Components.TestServer/RazorComponents/RemoteAuthenticationApp.razor new file mode 100644 index 000000000000..7cec534aca44 --- /dev/null +++ b/src/Components/test/testassets/Components.TestServer/RazorComponents/RemoteAuthenticationApp.razor @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/Components/test/testassets/Components.TestServer/RemoteAuthenticationStartup.cs b/src/Components/test/testassets/Components.TestServer/RemoteAuthenticationStartup.cs new file mode 100644 index 000000000000..69aca756bb9e --- /dev/null +++ b/src/Components/test/testassets/Components.TestServer/RemoteAuthenticationStartup.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Reflection; +using Components.TestServer.RazorComponents; +using Microsoft.AspNetCore.Mvc; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Tokens; + +namespace TestServer; + +public class RemoteAuthenticationStartup +{ + public void ConfigureServices(IServiceCollection services) + { + services.AddRazorComponents() + .AddInteractiveWebAssemblyComponents(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + app.Map("/subdir", app => + { + app.UseStaticFiles(); + app.UseRouting(); + app.UseAntiforgery(); + app.UseEndpoints(endpoints => + { + endpoints.MapRazorComponents() + .AddAdditionalAssemblies(Assembly.Load("Components.WasmRemoteAuthentication")) + .AddInteractiveWebAssemblyRenderMode(options => options.PathPrefix = "/WasmRemoteAuthentication"); + + var oidcEndpoints = endpoints.MapGroup("oidc"); + + // This is designed to test a single login at a time. + var issuer = ""; + oidcEndpoints.MapGet(".well-known/openid-configuration", (HttpRequest request, [FromHeader] string host) => + { + issuer = $"{(request.IsHttps ? "https" : "http")}://{host}"; + return Results.Json(new + { + issuer, + authorization_endpoint = $"{issuer}/subdir/oidc/authorize", + token_endpoint = $"{issuer}/subdir/oidc/token", + }); + }); + + var lastCode = ""; + oidcEndpoints.MapGet("authorize", (string redirect_uri, string? state, string? prompt, bool? preservedExtraQueryParams) => + { + // Require interaction so silent sign-in does not skip RedirectToLogin.razor. + if (prompt == "none") + { + return Results.Redirect($"{redirect_uri}?error=interaction_required&state={state}"); + } + + // Verify that the extra query parameters added by RedirectToLogin.razor are preserved. + if (preservedExtraQueryParams != true) + { + return Results.Redirect($"{redirect_uri}?error=invalid_request&error_description=extraQueryParams%20not%20preserved&state={state}"); + } + + lastCode = Random.Shared.Next().ToString(CultureInfo.InvariantCulture); + return Results.Redirect($"{redirect_uri}?code={lastCode}&state={state}"); + }); + + var jwtHandler = new JsonWebTokenHandler(); + oidcEndpoints.MapPost("token", ([FromForm] string code) => + { + if (string.IsNullOrEmpty(lastCode) && code != lastCode) + { + return Results.BadRequest("Bad code"); + } + + return Results.Json(new + { + token_type = "Bearer", + scope = "openid profile", + expires_in = 3600, + id_token = jwtHandler.CreateToken(new SecurityTokenDescriptor + { + Issuer = issuer, + Audience = "s6BhdRkqt3", + Claims = new Dictionary + { + ["sub"] = "248289761001", + ["name"] = "Jane Doe", + }, + }), + }); + }).DisableAntiforgery(); + }); + }); + } +} diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Components.WasmRemoteAuthentication.csproj b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Components.WasmRemoteAuthentication.csproj new file mode 100644 index 000000000000..3b44a098893d --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Components.WasmRemoteAuthentication.csproj @@ -0,0 +1,20 @@ + + + + $(DefaultNetCoreTargetFramework) + enable + enable + WasmRemoteAuthentication + + + + + <_BlazorBrotliCompressionLevel>NoCompression + + + + + + + + diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/Authentication.razor b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/Authentication.razor new file mode 100644 index 000000000000..9ec02f227154 --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/Authentication.razor @@ -0,0 +1,9 @@ +@page "/authentication/{action}" + +@using Microsoft.AspNetCore.Components.WebAssembly.Authentication + + + +@code { + [Parameter] public string? Action { get; set; } +} diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/TestRemoteAuthentication.razor b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/TestRemoteAuthentication.razor new file mode 100644 index 000000000000..a5a7cf05ec31 --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Pages/TestRemoteAuthentication.razor @@ -0,0 +1,13 @@ +@page "/test-remote-authentication" + +@using Microsoft.AspNetCore.Components.Authorization + + + +

Hello, @context.User.Identity?.Name!

+ + + @* Do this rather than rely on the [Authorize] attribute to avoid endpoint routing. *@ + + + diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Program.cs b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Program.cs new file mode 100644 index 000000000000..712d678c8920 --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Program.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.WebAssembly.Hosting; + +var builder = WebAssemblyHostBuilder.CreateDefault(args); + +builder.Services.AddOidcAuthentication(options => +{ + options.ProviderOptions.Authority = $"{builder.HostEnvironment.BaseAddress}oidc"; + options.ProviderOptions.ClientId = "s6BhdRkqt3"; + options.ProviderOptions.ResponseType = "code"; +}); + +await builder.Build().RunAsync(); diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Properties/launchSettings.json b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Properties/launchSettings.json new file mode 100644 index 000000000000..ab37532e31fb --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Properties/launchSettings.json @@ -0,0 +1,21 @@ +{ + "profiles": { + "http": { + "commandName": "Project", + "launchBrowser": true, + "inspectUri": "{wsProtocol}://{url.hostname}:{url.port}/_framework/debug/ws-proxy?browser={browserInspectUri}", + "applicationUrl": "http://localhost:5102", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:7293;http://localhost:5102", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/RedirectToLogin.razor b/src/Components/test/testassets/Components.WasmRemoteAuthentication/RedirectToLogin.razor new file mode 100644 index 000000000000..76739ecc85d4 --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/RedirectToLogin.razor @@ -0,0 +1,14 @@ +@using Microsoft.AspNetCore.Components.WebAssembly.Authentication + +@inject NavigationManager Navigation + +@code { + protected override void OnInitialized() + { + var request = new InteractiveRequestOptions { Interaction = InteractionType.SignIn, ReturnUrl = Navigation.Uri }; + var extraQueryParams = new Dictionary { ["preservedExtraQueryParams"] = "true" }; + request.TryAddAdditionalParameter("extraQueryParams", extraQueryParams); + + Navigation.NavigateToLogin("authentication/login", request); + } +} diff --git a/src/Components/test/testassets/Components.WasmRemoteAuthentication/Routes.razor b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Routes.razor new file mode 100644 index 000000000000..b69e64ec3e62 --- /dev/null +++ b/src/Components/test/testassets/Components.WasmRemoteAuthentication/Routes.razor @@ -0,0 +1,27 @@ +@using Microsoft.AspNetCore.Components.Authorization +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Web + + + + + + + @if (context.User.Identity?.IsAuthenticated != true) + { + + } + else + { +

You are not authorized to access this resource.

+ } + + + + + + Not found +

Sorry, there's nothing at this address.

+ + + diff --git a/src/DataProtection/CreateTestCert.ps1 b/src/DataProtection/CreateTestCert.ps1 index a85a040f05b6..5c97f792bcfc 100644 --- a/src/DataProtection/CreateTestCert.ps1 +++ b/src/DataProtection/CreateTestCert.ps1 @@ -1,3 +1,5 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification='dev utility tool')] + # # Generates a new test cert in a .pfx file # Obviously, don't actually use this to produce production certs diff --git a/src/DataProtection/DataProtection/src/DataProtectionServiceCollectionExtensions.cs b/src/DataProtection/DataProtection/src/DataProtectionServiceCollectionExtensions.cs index 9a63abe0e1c3..5441d535db0c 100644 --- a/src/DataProtection/DataProtection/src/DataProtectionServiceCollectionExtensions.cs +++ b/src/DataProtection/DataProtection/src/DataProtectionServiceCollectionExtensions.cs @@ -67,6 +67,8 @@ private static void AddDataProtectionServices(IServiceCollection services) services.TryAddEnumerable( ServiceDescriptor.Singleton, KeyManagementOptionsSetup>()); + services.TryAddEnumerable( + ServiceDescriptor.Singleton, KeyManagementOptionsPostSetup>()); services.TryAddEnumerable( ServiceDescriptor.Transient, DataProtectionOptionsSetup>()); diff --git a/src/DataProtection/DataProtection/src/Internal/DefaultTypeNameResolver.cs b/src/DataProtection/DataProtection/src/Internal/DefaultTypeNameResolver.cs new file mode 100644 index 000000000000..40a63826048a --- /dev/null +++ b/src/DataProtection/DataProtection/src/Internal/DefaultTypeNameResolver.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.AspNetCore.DataProtection.Internal; + +internal sealed class DefaultTypeNameResolver : ITypeNameResolver +{ + public static readonly DefaultTypeNameResolver Instance = new(); + + private DefaultTypeNameResolver() + { + } + + [UnconditionalSuppressMessage("Trimmer", "IL2057", Justification = "Type.GetType is only used to resolve statically known types that are referenced by DataProtection assembly.")] + public bool TryResolveType(string typeName, [NotNullWhen(true)] out Type? type) + { + try + { + // Some exceptions are thrown regardless of the value of throwOnError. + // For example, if the type is found but cannot be loaded, + // a System.TypeLoadException is thrown even if throwOnError is false. + type = Type.GetType(typeName, throwOnError: false); + return type != null; + } + catch + { + type = null; + return false; + } + } +} diff --git a/src/DataProtection/DataProtection/src/Internal/ITypeNameResolver.cs b/src/DataProtection/DataProtection/src/Internal/ITypeNameResolver.cs new file mode 100644 index 000000000000..f31a037201b8 --- /dev/null +++ b/src/DataProtection/DataProtection/src/Internal/ITypeNameResolver.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.AspNetCore.DataProtection.Internal; + +internal interface ITypeNameResolver +{ + bool TryResolveType(string typeName, [NotNullWhen(true)] out Type? type); +} diff --git a/src/DataProtection/DataProtection/src/Internal/KeyManagementOptionsPostSetup.cs b/src/DataProtection/DataProtection/src/Internal/KeyManagementOptionsPostSetup.cs new file mode 100644 index 000000000000..95f1698a9d47 --- /dev/null +++ b/src/DataProtection/DataProtection/src/Internal/KeyManagementOptionsPostSetup.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Xml.Linq; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using Microsoft.AspNetCore.DataProtection.Repositories; +using Microsoft.AspNetCore.DataProtection.XmlEncryption; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.AspNetCore.DataProtection.Internal; + +/// +/// Performs additional configuration, after the user's configuration has been applied. +/// +/// +/// In practice, this type is used to set key management to readonly mode if an environment variable is set and the user +/// has not explicitly configured data protection. +/// +internal sealed class KeyManagementOptionsPostSetup : IPostConfigureOptions +{ + /// + /// Settable as `ReadOnlyDataProtectionKeyDirectory`, `DOTNET_ReadOnlyDataProtectionKeyDirectory`, + /// or `ASPNETCORE_ReadOnlyDataProtectionKeyDirectory`, in descending order of precedence. + /// + internal const string ReadOnlyDataProtectionKeyDirectoryKey = "ReadOnlyDataProtectionKeyDirectory"; + + private readonly string? _keyDirectoryPath; + private readonly ILoggerFactory? _loggerFactory; // Null iff _keyDirectoryPath is null + private readonly ILogger? _logger; // Null iff _keyDirectoryPath is null + + public KeyManagementOptionsPostSetup() + { + // If there's no IConfiguration, there's no _keyDirectoryPath and this type will do nothing. + // This is mostly a convenience for tests since ASP.NET Core apps will have an IConfiguration. + } + + public KeyManagementOptionsPostSetup(IConfiguration configuration, ILoggerFactory loggerFactory) + { + var dirPath = configuration[ReadOnlyDataProtectionKeyDirectoryKey]; + if (string.IsNullOrEmpty(dirPath)) + { + return; + } + + _keyDirectoryPath = dirPath; + _loggerFactory = loggerFactory; + _logger = loggerFactory.CreateLogger(); + } + + void IPostConfigureOptions.PostConfigure(string? name, KeyManagementOptions options) + { + if (_keyDirectoryPath is null) + { + // There's no logger, so we couldn't log if we wanted to + return; + } + + var logger = _logger!; + + if (name != Options.DefaultName) + { + logger.IgnoringReadOnlyConfigurationForNonDefaultOptions(ReadOnlyDataProtectionKeyDirectoryKey, name); + return; + } + + // If Data Protection has not been configured, then set it up according to the environment variable + if (options is { XmlRepository: null, XmlEncryptor: null }) + { + var keyDirectory = new DirectoryInfo(_keyDirectoryPath); + + logger.UsingReadOnlyKeyConfiguration(keyDirectory.FullName); + + options.AutoGenerateKeys = false; + options.XmlEncryptor = InvalidEncryptor.Instance; + options.XmlRepository = new ReadOnlyFileSystemXmlRepository(keyDirectory, _loggerFactory!); + } + else if (options.XmlRepository is not null) + { + logger.NotUsingReadOnlyKeyConfigurationBecauseOfRepository(); + } + else + { + logger.NotUsingReadOnlyKeyConfigurationBecauseOfEncryptor(); + } + } + + private sealed class InvalidEncryptor : IXmlEncryptor + { + public static readonly IXmlEncryptor Instance = new InvalidEncryptor(); + + private InvalidEncryptor() + { + } + + EncryptedXmlInfo IXmlEncryptor.Encrypt(XElement plaintextElement) + { + throw new InvalidOperationException("Keys access is set up as read-only, so nothing should be encrypting"); + } + } + + private sealed class ReadOnlyFileSystemXmlRepository : FileSystemXmlRepository + { + public ReadOnlyFileSystemXmlRepository(DirectoryInfo directory, ILoggerFactory loggerFactory) + : base(directory, loggerFactory) + { + } + + public override void StoreElement(XElement element, string friendlyName) + { + throw new InvalidOperationException("Keys access is set up as read-only, so nothing should be storing keys"); + } + } +} diff --git a/src/DataProtection/DataProtection/src/KeyManagement/XmlKeyManager.cs b/src/DataProtection/DataProtection/src/KeyManagement/XmlKeyManager.cs index 57a678e6480b..9549d8d17721 100644 --- a/src/DataProtection/DataProtection/src/KeyManagement/XmlKeyManager.cs +++ b/src/DataProtection/DataProtection/src/KeyManagement/XmlKeyManager.cs @@ -49,6 +49,7 @@ public sealed class XmlKeyManager : IKeyManager, IInternalXmlKeyManager private const string RevokeAllKeysValue = "*"; private readonly IActivator _activator; + private readonly ITypeNameResolver _typeNameResolver; private readonly AlgorithmConfiguration _authenticatedEncryptorConfiguration; private readonly IKeyEscrowSink? _keyEscrowSink; private readonly IInternalXmlKeyManager _internalKeyManager; @@ -112,6 +113,8 @@ internal XmlKeyManager( var escrowSinks = keyManagementOptions.Value.KeyEscrowSinks; _keyEscrowSink = escrowSinks.Count > 0 ? new AggregateKeyEscrowSink(escrowSinks) : null; _activator = activator; + // Note: ITypeNameResolver is only implemented on the activator in tests. In production, it's always DefaultTypeNameResolver. + _typeNameResolver = activator as ITypeNameResolver ?? DefaultTypeNameResolver.Instance; TriggerAndResetCacheExpirationToken(suppressLogging: true); _internalKeyManager = _internalKeyManager ?? this; _encryptorFactories = keyManagementOptions.Value.AuthenticatedEncryptorFactories; @@ -460,27 +463,27 @@ IAuthenticatedEncryptorDescriptor IInternalXmlKeyManager.DeserializeDescriptorFr } } - [UnconditionalSuppressMessage("Trimmer", "IL2057", Justification = "Type.GetType result is only useful with types that are referenced by DataProtection assembly.")] private IAuthenticatedEncryptorDescriptorDeserializer CreateDeserializer(string descriptorDeserializerTypeName) { - var resolvedTypeName = TypeForwardingActivator.TryForwardTypeName(descriptorDeserializerTypeName, out var forwardedTypeName) + // typeNameToMatch will be used for matching against known types but not passed to the activator. + // The activator will do its own forwarding. + var typeNameToMatch = TypeForwardingActivator.TryForwardTypeName(descriptorDeserializerTypeName, out var forwardedTypeName) ? forwardedTypeName : descriptorDeserializerTypeName; - var type = Type.GetType(resolvedTypeName, throwOnError: false); - if (type == typeof(AuthenticatedEncryptorDescriptorDeserializer)) + if (typeof(AuthenticatedEncryptorDescriptorDeserializer).MatchName(typeNameToMatch, _typeNameResolver)) { return _activator.CreateInstance(descriptorDeserializerTypeName); } - else if (type == typeof(CngCbcAuthenticatedEncryptorDescriptorDeserializer) && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && typeof(CngCbcAuthenticatedEncryptorDescriptorDeserializer).MatchName(typeNameToMatch, _typeNameResolver)) { return _activator.CreateInstance(descriptorDeserializerTypeName); } - else if (type == typeof(CngGcmAuthenticatedEncryptorDescriptorDeserializer) && RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && typeof(CngGcmAuthenticatedEncryptorDescriptorDeserializer).MatchName(typeNameToMatch, _typeNameResolver)) { return _activator.CreateInstance(descriptorDeserializerTypeName); } - else if (type == typeof(ManagedAuthenticatedEncryptorDescriptorDeserializer)) + else if (typeof(ManagedAuthenticatedEncryptorDescriptorDeserializer).MatchName(typeNameToMatch, _typeNameResolver)) { return _activator.CreateInstance(descriptorDeserializerTypeName); } diff --git a/src/DataProtection/DataProtection/src/LoggingExtensions.cs b/src/DataProtection/DataProtection/src/LoggingExtensions.cs index 701ce885c79c..3522725b837b 100644 --- a/src/DataProtection/DataProtection/src/LoggingExtensions.cs +++ b/src/DataProtection/DataProtection/src/LoggingExtensions.cs @@ -237,4 +237,16 @@ private static bool IsLogLevelEnabledCore([NotNullWhen(true)] ILogger? logger, L [LoggerMessage(60, LogLevel.Warning, "Storing keys in a directory '{path}' that may not be persisted outside of the container. Protected data will be unavailable when container is destroyed. For more information go to https://aka.ms/aspnet/dataprotectionwarning", EventName = "UsingEphemeralFileSystemLocationInContainer")] public static partial void UsingEphemeralFileSystemLocationInContainer(this ILogger logger, string path); + + [LoggerMessage(61, LogLevel.Trace, "Ignoring configuration '{PropertyName}' for options instance '{OptionsName}'", EventName = "IgnoringReadOnlyConfigurationForNonDefaultOptions")] + public static partial void IgnoringReadOnlyConfigurationForNonDefaultOptions(this ILogger logger, string propertyName, string? optionsName); + + [LoggerMessage(62, LogLevel.Information, "Enabling read-only key access with repository directory '{Path}'", EventName = "UsingReadOnlyKeyConfiguration")] + public static partial void UsingReadOnlyKeyConfiguration(this ILogger logger, string path); + + [LoggerMessage(63, LogLevel.Debug, "Not enabling read-only key access because an XML repository has been specified", EventName = "NotUsingReadOnlyKeyConfigurationBecauseOfRepository")] + public static partial void NotUsingReadOnlyKeyConfigurationBecauseOfRepository(this ILogger logger); + + [LoggerMessage(64, LogLevel.Debug, "Not enabling read-only key access because an XML encryptor has been specified", EventName = "NotUsingReadOnlyKeyConfigurationBecauseOfEncryptor")] + public static partial void NotUsingReadOnlyKeyConfigurationBecauseOfEncryptor(this ILogger logger); } diff --git a/src/DataProtection/DataProtection/src/TypeExtensions.cs b/src/DataProtection/DataProtection/src/TypeExtensions.cs index 89b69d0b70db..1766ff6a7c2d 100644 --- a/src/DataProtection/DataProtection/src/TypeExtensions.cs +++ b/src/DataProtection/DataProtection/src/TypeExtensions.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics.CodeAnalysis; +using Microsoft.AspNetCore.DataProtection.Internal; namespace Microsoft.AspNetCore.DataProtection; @@ -39,4 +40,16 @@ public static Type GetTypeWithTrimFriendlyErrorMessage(string typeName) throw new InvalidOperationException($"Unable to load type '{typeName}'. If the app is published with trimming then this type may have been trimmed. Ensure the type's assembly is excluded from trimming.", ex); } } + + public static bool MatchName(this Type matchType, string resolvedTypeName, ITypeNameResolver typeNameResolver) + { + // Before attempting to resolve the name to a type, check if it starts with the full name of the type. + // Use StartsWith to ignore potential assembly version differences. + if (matchType.FullName != null && resolvedTypeName.StartsWith(matchType.FullName, StringComparison.Ordinal)) + { + return typeNameResolver.TryResolveType(resolvedTypeName, out var resolvedType) && resolvedType == matchType; + } + + return false; + } } diff --git a/src/DataProtection/DataProtection/src/XmlEncryption/XmlEncryptionExtensions.cs b/src/DataProtection/DataProtection/src/XmlEncryption/XmlEncryptionExtensions.cs index 1b99664b486c..62d1bdf0b99c 100644 --- a/src/DataProtection/DataProtection/src/XmlEncryption/XmlEncryptionExtensions.cs +++ b/src/DataProtection/DataProtection/src/XmlEncryption/XmlEncryptionExtensions.cs @@ -67,27 +67,30 @@ public static XElement DecryptElement(this XElement element, IActivator activato return doc.Root!; } - [UnconditionalSuppressMessage("Trimmer", "IL2057", Justification = "Type.GetType result is only useful with types that are referenced by DataProtection assembly.")] private static IXmlDecryptor CreateDecryptor(IActivator activator, string decryptorTypeName) { - var resolvedTypeName = TypeForwardingActivator.TryForwardTypeName(decryptorTypeName, out var forwardedTypeName) + // typeNameToMatch will be used for matching against known types but not passed to the activator. + // The activator will do its own forwarding. + var typeNameToMatch = TypeForwardingActivator.TryForwardTypeName(decryptorTypeName, out var forwardedTypeName) ? forwardedTypeName : decryptorTypeName; - var type = Type.GetType(resolvedTypeName, throwOnError: false); - if (type == typeof(DpapiNGXmlDecryptor)) + // Note: ITypeNameResolver is only implemented on the activator in tests. In production, it's always DefaultTypeNameResolver. + var typeNameResolver = activator as ITypeNameResolver ?? DefaultTypeNameResolver.Instance; + + if (typeof(DpapiNGXmlDecryptor).MatchName(typeNameToMatch, typeNameResolver)) { return activator.CreateInstance(decryptorTypeName); } - else if (type == typeof(DpapiXmlDecryptor)) + else if (typeof(DpapiXmlDecryptor).MatchName(typeNameToMatch, typeNameResolver)) { return activator.CreateInstance(decryptorTypeName); } - else if (type == typeof(EncryptedXmlDecryptor)) + else if (typeof(EncryptedXmlDecryptor).MatchName(typeNameToMatch, typeNameResolver)) { return activator.CreateInstance(decryptorTypeName); } - else if (type == typeof(NullXmlDecryptor)) + else if (typeof(NullXmlDecryptor).MatchName(typeNameToMatch, typeNameResolver)) { return activator.CreateInstance(decryptorTypeName); } diff --git a/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/Internal/KeyManagementOptionsPostSetupTest.cs b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/Internal/KeyManagementOptionsPostSetupTest.cs new file mode 100644 index 000000000000..33906ffeca03 --- /dev/null +++ b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/Internal/KeyManagementOptionsPostSetupTest.cs @@ -0,0 +1,197 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Xml.Linq; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using Microsoft.AspNetCore.DataProtection.Repositories; +using Microsoft.AspNetCore.DataProtection.XmlEncryption; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace Microsoft.AspNetCore.DataProtection.Internal; + +public class KeyManagementOptionsPostSetupTest +{ + private static readonly string keyDir = new DirectoryInfo("/testpath").FullName; + private static readonly XElement xElement = new("element"); + + [Fact] + public void ConfigureReadOnly() + { + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, keyDir), + ]).Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var options = new KeyManagementOptions(); + + setup.PostConfigure(Options.DefaultName, options); + + AssertReadOnly(options, keyDir); + } + + [Fact] + public void ConfigureReadOnly_NonDefaultInstance() + { + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, keyDir), + ]).Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var options = new KeyManagementOptions(); + + setup.PostConfigure(Options.DefaultName + 1, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + + [Fact] + public void ConfigureReadOnly_EmptyDirPath() + { + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, ""), + ]).Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var options = new KeyManagementOptions(); + + setup.PostConfigure(Options.DefaultName, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + + [Fact] + public void ConfigureReadOnly_ExplicitRepository() + { + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, keyDir), + ]).Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var xmlDir = Directory.CreateTempSubdirectory(); + try + { + var options = new KeyManagementOptions() + { + XmlRepository = new FileSystemXmlRepository(xmlDir, NullLoggerFactory.Instance), + }; + + setup.PostConfigure(Options.DefaultName, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + finally + { + xmlDir.Delete(recursive: true); + } + } + + [Fact] + public void ConfigureReadOnly_ExplicitEncryptor() + { + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, keyDir), + ]).Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var options = new KeyManagementOptions() + { + XmlEncryptor = new NullXmlEncryptor(), + }; + + setup.PostConfigure(Options.DefaultName, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + + [Fact] + public void NotConfigured_NoProperty() + { + var config = new ConfigurationBuilder().AddInMemoryCollection().Build(); + + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(config, NullLoggerFactory.Instance); + + var options = new KeyManagementOptions(); + + setup.PostConfigure(Options.DefaultName, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + + [Fact] + public void NotConfigured_NoIConfiguration() + { + IPostConfigureOptions setup = new KeyManagementOptionsPostSetup(); + + var options = new KeyManagementOptions(); + + setup.PostConfigure(Options.DefaultName, options); + + AssertNotReadOnly(options, keyDir); + + Assert.True(options.AutoGenerateKeys); + } + + private static void AssertReadOnly(KeyManagementOptions options, string keyDir) + { + // Effect 1: No key generation + Assert.False(options.AutoGenerateKeys); + + var repository = options.XmlRepository as FileSystemXmlRepository; + Assert.NotNull(repository); + + // Effect 2: Location from configuration + Assert.Equal(keyDir, repository.Directory.FullName); + + // Effect 3: No writing + Assert.Throws(() => repository.StoreElement(xElement, friendlyName: null)); + + // Effect 4: No key encryption + Assert.NotNull(options.XmlEncryptor); + Assert.Throws(() => options.XmlEncryptor.Encrypt(xElement)); + } + + private static void AssertNotReadOnly(KeyManagementOptions options, string keyDir) + { + // Missing effect 1: No key generation + Assert.True(options.AutoGenerateKeys); + + var repository = options.XmlRepository; + if (repository is not null) + { + // Missing effect 2: Location from configuration + Assert.NotEqual(keyDir, (repository as FileSystemXmlRepository)?.Directory.FullName); + + // Missing effect 3: No writing + repository.StoreElement(xElement, friendlyName: null); + } + + var encryptor = options.XmlEncryptor; + if (encryptor is not null) + { + // Missing effect 4: No key encryption + options.XmlEncryptor.Encrypt(xElement); + } + } +} diff --git a/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/ServiceCollectionTests.cs b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/ServiceCollectionTests.cs index c21d33d169f5..010c455bd70e 100644 --- a/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/ServiceCollectionTests.cs +++ b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/ServiceCollectionTests.cs @@ -1,8 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Xml.Linq; +using Microsoft.AspNetCore.DataProtection.Internal; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using Microsoft.AspNetCore.DataProtection.Repositories; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; namespace Microsoft.AspNetCore.DataProtection; @@ -61,4 +67,62 @@ public void CanResolveAllRegisteredServices() Assert.NotNull(services.GetService(descriptor.ServiceType)); } } + + [Fact] + public void ReadOnlyDataProtectionKeyDirectory() + { + var keyDir = new DirectoryInfo("/testpath").FullName; + + var config = new ConfigurationBuilder().AddInMemoryCollection( + [ + new KeyValuePair(KeyManagementOptionsPostSetup.ReadOnlyDataProtectionKeyDirectoryKey, keyDir), + ]).Build(); + + var services = new ServiceCollection() + .AddSingleton(config) + .AddSingleton(NullLoggerFactory.Instance) + .AddDataProtection() + .Services + .BuildServiceProvider(); + + var options = services.GetRequiredService>().Value; + + // Effect 1: No key generation + Assert.False(options.AutoGenerateKeys); + + var repository = options.XmlRepository as FileSystemXmlRepository; + Assert.NotNull(repository); + + // Effect 2: Location from configuration + Assert.Equal(keyDir, repository.Directory.FullName); + + var xElement = new XElement("element"); + + // Effect 3: No writing + Assert.Throws(() => repository.StoreElement(xElement, friendlyName: null)); + + // Effect 4: No key encryption + Assert.NotNull(options.XmlEncryptor); + Assert.Throws(() => options.XmlEncryptor.Encrypt(xElement)); + } + + [Fact] + public void NoReadOnlyDataProtectionKeyDirectory() + { + var config = new ConfigurationBuilder().AddInMemoryCollection().Build(); + + var services = new ServiceCollection() + .AddSingleton(config) + .AddSingleton(NullLoggerFactory.Instance) + .AddDataProtection() + .Services + .BuildServiceProvider(); + + var options = services.GetRequiredService>().Value; + + // Missing effect 1: No key generation + Assert.True(options.AutoGenerateKeys); + + // KeyManagementOptionsPostSetupTest covers other missing effects + } } diff --git a/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/XmlEncryption/XmlEncryptionExtensionsTests.cs b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/XmlEncryption/XmlEncryptionExtensionsTests.cs index dffa10477e87..2897f4b46182 100644 --- a/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/XmlEncryption/XmlEncryptionExtensionsTests.cs +++ b/src/DataProtection/DataProtection/test/Microsoft.AspNetCore.DataProtection.Tests/XmlEncryption/XmlEncryptionExtensionsTests.cs @@ -49,6 +49,100 @@ public void DecryptElement_RootNodeRequiresDecryption_Success() XmlAssert.Equal("", retVal); } + [Fact] + public void DecryptElement_CustomType_TypeNameResolverNotCalled() + { + // Arrange + var decryptorTypeName = typeof(MyXmlDecryptor).AssemblyQualifiedName; + + var original = XElement.Parse(@$" + + + "); + + var mockActivator = new Mock(); + mockActivator.ReturnDecryptedElementGivenDecryptorTypeNameAndInput(decryptorTypeName, "", ""); + var mockTypeNameResolver = mockActivator.As(); + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSingleton(mockActivator.Object); + var services = serviceCollection.BuildServiceProvider(); + var activator = services.GetActivator(); + + // Act + var retVal = original.DecryptElement(activator); + + // Assert + XmlAssert.Equal("", retVal); + Type resolvedType; + mockTypeNameResolver.Verify(o => o.TryResolveType(It.IsAny(), out resolvedType), Times.Never()); + } + + [Fact] + public void DecryptElement_KnownType_TypeNameResolverCalled() + { + // Arrange + var decryptorTypeName = typeof(NullXmlDecryptor).AssemblyQualifiedName; + TypeForwardingActivator.TryForwardTypeName(decryptorTypeName, out var forwardedTypeName); + + var original = XElement.Parse(@$" + + + + + "); + + var mockActivator = new Mock(); + mockActivator.Setup(o => o.CreateInstance(typeof(NullXmlDecryptor), decryptorTypeName)).Returns(new NullXmlDecryptor()); + var mockTypeNameResolver = mockActivator.As(); + var resolvedType = typeof(NullXmlDecryptor); + mockTypeNameResolver.Setup(mockTypeNameResolver => mockTypeNameResolver.TryResolveType(forwardedTypeName, out resolvedType)).Returns(true); + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSingleton(mockActivator.Object); + var services = serviceCollection.BuildServiceProvider(); + var activator = services.GetActivator(); + + // Act + var retVal = original.DecryptElement(activator); + + // Assert + XmlAssert.Equal("", retVal); + mockTypeNameResolver.Verify(o => o.TryResolveType(It.IsAny(), out resolvedType), Times.Once()); + } + + [Fact] + public void DecryptElement_KnownType_UnableToResolveType_Success() + { + // Arrange + var decryptorTypeName = typeof(NullXmlDecryptor).AssemblyQualifiedName; + + var original = XElement.Parse(@$" + + + + + "); + + var mockActivator = new Mock(); + mockActivator.Setup(o => o.CreateInstance(typeof(IXmlDecryptor), decryptorTypeName)).Returns(new NullXmlDecryptor()); + var mockTypeNameResolver = mockActivator.As(); + Type resolvedType = null; + mockTypeNameResolver.Setup(mockTypeNameResolver => mockTypeNameResolver.TryResolveType(It.IsAny(), out resolvedType)).Returns(false); + + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSingleton(mockActivator.Object); + var services = serviceCollection.BuildServiceProvider(); + var activator = services.GetActivator(); + + // Act + var retVal = original.DecryptElement(activator); + + // Assert + XmlAssert.Equal("", retVal); + mockTypeNameResolver.Verify(o => o.TryResolveType(It.IsAny(), out resolvedType), Times.Once()); + } + [Fact] public void DecryptElement_MultipleNodesRequireDecryption_AvoidsRecursion_Success() { diff --git a/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/RouteStringSyntaxDetector.cs b/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/RouteStringSyntaxDetector.cs index 345336918ea8..d5814e002b5b 100644 --- a/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/RouteStringSyntaxDetector.cs +++ b/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/RouteStringSyntaxDetector.cs @@ -149,6 +149,19 @@ private static bool HasLanguageComment( return true; } + // Check for the common case of a string literal in a large binary expression. For example `"..." + "..." + + // "..."` We never want to consider these as regex/json tokens as processing them would require knowing the + // contents of every string literal, and having our lexers/parsers somehow stitch them all together. This is + // beyond what those systems support (and would only work for constant strings anyways). This prevents both + // incorrect results *and* avoids heavy perf hits walking up large binary expressions (often while a caller is + // themselves walking down such a large expression). + if (token.Parent.IsLiteralExpression() && + token.Parent.Parent.IsBinaryExpression() && + token.Parent.Parent.RawKind == (int)SyntaxKind.AddExpression) + { + return false; + } + for (var node = token.Parent; node != null; node = node.Parent) { if (HasLanguageComment(node.GetLeadingTrivia(), out identifier, out options)) diff --git a/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/SyntaxNodeExtensions.cs b/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/SyntaxNodeExtensions.cs index 525854a63eb4..3eba331a5ffb 100644 --- a/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/SyntaxNodeExtensions.cs +++ b/src/Framework/AspNetCoreAnalyzers/src/Analyzers/RouteEmbeddedLanguage/Infrastructure/SyntaxNodeExtensions.cs @@ -33,6 +33,12 @@ public static SyntaxNode GetRequiredParent(this SyntaxNode node) return parent; } + public static bool IsLiteralExpression([NotNullWhen(true)] this SyntaxNode? node) + => node is LiteralExpressionSyntax; + + public static bool IsBinaryExpression([NotNullWhen(true)] this SyntaxNode? node) + => node is BinaryExpressionSyntax; + [return: NotNullIfNotNull("node")] public static SyntaxNode? WalkUpParentheses(this SyntaxNode? node) { diff --git a/src/Framework/AspNetCoreAnalyzers/test/Microsoft.AspNetCore.App.Analyzers.Test.csproj b/src/Framework/AspNetCoreAnalyzers/test/Microsoft.AspNetCore.App.Analyzers.Test.csproj index 5f223d266110..8070e80c0b51 100644 --- a/src/Framework/AspNetCoreAnalyzers/test/Microsoft.AspNetCore.App.Analyzers.Test.csproj +++ b/src/Framework/AspNetCoreAnalyzers/test/Microsoft.AspNetCore.App.Analyzers.Test.csproj @@ -1,4 +1,4 @@ - + $(DefaultNetCoreTargetFramework) diff --git a/src/Framework/AspNetCoreAnalyzers/test/RouteEmbeddedLanguage/RoutePatternAnalyzerTests.cs b/src/Framework/AspNetCoreAnalyzers/test/RouteEmbeddedLanguage/RoutePatternAnalyzerTests.cs index 921e7f7f5d2f..a4c06cd0a261 100644 --- a/src/Framework/AspNetCoreAnalyzers/test/RouteEmbeddedLanguage/RoutePatternAnalyzerTests.cs +++ b/src/Framework/AspNetCoreAnalyzers/test/RouteEmbeddedLanguage/RoutePatternAnalyzerTests.cs @@ -1,17 +1,28 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using System.Globalization; +using System.Text; using Microsoft.AspNetCore.Analyzer.Testing; using Microsoft.AspNetCore.Analyzers.RenderTreeBuilder; using Microsoft.AspNetCore.Analyzers.RouteEmbeddedLanguage.Infrastructure; +using Microsoft.CodeAnalysis; +using Xunit.Abstractions; namespace Microsoft.AspNetCore.Analyzers.RouteEmbeddedLanguage; public partial class RoutePatternAnalyzerTests { + private readonly ITestOutputHelper _testOutputHelper; + private TestDiagnosticAnalyzerRunner Runner { get; } = new(new RoutePatternAnalyzer()); + public RoutePatternAnalyzerTests(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + } + [Fact] public async Task CommentOnString_ReportResults() { @@ -512,4 +523,125 @@ public object TestAction(int id) // Assert Assert.Empty(diagnostics); } + + [Fact] + public async Task ConcatString_PerformanceTest() + { + // Arrange + var builder = new StringBuilder(); + builder.AppendLine(""" + class Program + { + static void Main() { } + static readonly string _s = + """); + for (var i = 0; i < 2000; i++) + { + builder.AppendLine(" \"a{}bc\" +"); + } + builder.AppendLine(""" + ""; + } + """); + var source = TestSource.Read(builder.ToString()); + + // Act 1 + // Warm up. + var diagnostics1 = await Runner.GetDiagnosticsAsync(source.Source); + + // Assert 1 + Assert.Empty(diagnostics1); + + // Act 2 + // Measure analysis. + var stopwatch = Stopwatch.StartNew(); + + var diagnostics2 = await Runner.GetDiagnosticsAsync(source.Source); + _testOutputHelper.WriteLine($"Elapsed time: {stopwatch.Elapsed}"); + + // Assert 2 + Assert.Empty(diagnostics2); + } + + [Fact] + public async Task ConcatString_DetectLanguage_NoWarningsBecauseConcatString() + { + // Arrange + var builder = new StringBuilder(); + builder.AppendLine(""" + class Program + { + static void Main() { } + // lang=Route + static readonly string _s = + """); + for (var i = 0; i < 2000; i++) + { + builder.AppendLine(" \"a{}bc\" +"); + } + builder.AppendLine(""" + ""; + } + """); + var source = TestSource.Read(builder.ToString()); + + // Act + var diagnostics = await Runner.GetDiagnosticsAsync(source.Source); + + // Assert + Assert.Empty(diagnostics); + } + + [Fact] + public async Task NestedLangComment_NoWarningsBecauseConcatString() + { + // Arrange + var builder = new StringBuilder(); + builder.AppendLine(""" + class Program + { + static void Main() { } + static readonly string _s = + "{/*MM0*/te*st0}" + + // lang=Route + "{/*MM1*/te*st1}" + + "{/*MM2*/te*st2}" + + "{test3}"; + } + """); + var source = TestSource.Read(builder.ToString()); + + // Act + var diagnostics = await Runner.GetDiagnosticsAsync(source.Source); + + // Assert + Assert.Empty(diagnostics); + } + + [Fact] + public async Task TopLangComment_NoWarningsBecauseConcatString() + { + // Arrange + var builder = new StringBuilder(); + builder.AppendLine(""" + class Program + { + static void Main() { } + static readonly string _s = + // lang=Route + "{/*MM0*/te*st0}" + + "{/*MM1*/te*st1}" + + "{/*MM2*/te*st2}" + + // lang=regex + "{test3}"; + } + """); + var source = TestSource.Read(builder.ToString()); + + // Act + var diagnostics = await Runner.GetDiagnosticsAsync(source.Source); + + // Assert + Assert.Empty(diagnostics); + } } diff --git a/src/Hosting/Server.IntegrationTesting/src/Deployers/RemoteWindowsDeployer/RemotePSSessionHelper.ps1 b/src/Hosting/Server.IntegrationTesting/src/Deployers/RemoteWindowsDeployer/RemotePSSessionHelper.ps1 index e5c54d21e810..62d6aef70f5d 100644 --- a/src/Hosting/Server.IntegrationTesting/src/Deployers/RemoteWindowsDeployer/RemotePSSessionHelper.ps1 +++ b/src/Hosting/Server.IntegrationTesting/src/Deployers/RemoteWindowsDeployer/RemotePSSessionHelper.ps1 @@ -1,4 +1,6 @@ -[CmdletBinding()] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification='dev utility tool')] + +[CmdletBinding()] param( [Parameter(Mandatory=$true)] [string]$serverName, diff --git a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheOptions.cs b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheOptions.cs index a13262e03c28..ec42d5a5e16c 100644 --- a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheOptions.cs +++ b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheOptions.cs @@ -54,18 +54,13 @@ static bool GetDefaultValue() => set => _useForceReconnect = value; } - internal ConfigurationOptions GetConfiguredOptions(string libSuffix) + internal ConfigurationOptions GetConfiguredOptions() { - var options = ConfigurationOptions?.Clone() ?? ConfigurationOptions.Parse(Configuration!); + var options = ConfigurationOptions ?? ConfigurationOptions.Parse(Configuration!); // we don't want an initially unavailable server to prevent DI creating the service itself options.AbortOnConnectFail = false; - if (!string.IsNullOrWhiteSpace(libSuffix)) - { - var provider = DefaultOptionsProvider.GetProvider(options.EndPoints); - options.LibraryName = $"{provider.LibraryName} {libSuffix}"; - } return options; } } diff --git a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.Log.cs b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.Log.cs index adc254675c10..c80167695ebf 100644 --- a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.Log.cs +++ b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.Log.cs @@ -14,4 +14,7 @@ internal partial class RedisOutputCacheStore [LoggerMessage(2, LogLevel.Error, "Fatal error occurred executing redis output-cache GC loop.", EventName = "RedisOutputCacheGCFatalError")] internal static partial void RedisOutputCacheGCFatalError(ILogger logger, Exception exception); + + [LoggerMessage(3, LogLevel.Debug, "Unable to add library name suffix.", EventName = "UnableToAddLibraryNameSuffix")] + internal static partial void UnableToAddLibraryNameSuffix(ILogger logger, Exception exception); } diff --git a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.cs b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.cs index 3b8f1d21670b..1dc30df00f2c 100644 --- a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.cs +++ b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/src/RedisOutputCacheStore.cs @@ -332,7 +332,7 @@ private async ValueTask ConnectSlowAsync(CancellationToken token) IConnectionMultiplexer connection; if (_options.ConnectionMultiplexerFactory is null) { - connection = await ConnectionMultiplexer.ConnectAsync(_options.GetConfiguredOptions("asp.net OC")).ConfigureAwait(false); + connection = await ConnectionMultiplexer.ConnectAsync(_options.GetConfiguredOptions()).ConfigureAwait(false); } else { @@ -415,6 +415,7 @@ private void PrepareConnection(IConnectionMultiplexer connection) WriteTimeTicks(ref _lastConnectTicks, DateTimeOffset.UtcNow); ValidateServerFeatures(connection); TryRegisterProfiler(connection); + TryAddSuffix(connection); } private void ValidateServerFeatures(IConnectionMultiplexer connection) @@ -451,6 +452,19 @@ private void TryRegisterProfiler(IConnectionMultiplexer connection) } } + private void TryAddSuffix(IConnectionMultiplexer connection) + { + try + { + connection.AddLibraryNameSuffix("aspnet"); + connection.AddLibraryNameSuffix("OC"); + } + catch (Exception ex) + { + UnableToAddLibraryNameSuffix(_logger, ex); + } + } + private static void WriteTimeTicks(ref long field, DateTimeOffset value) { var ticks = value == DateTimeOffset.MinValue ? 0L : value.UtcTicks; diff --git a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/test/RedisConnectionFixture.cs b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/test/RedisConnectionFixture.cs index 23f379825dce..d7ea5529b41c 100644 --- a/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/test/RedisConnectionFixture.cs +++ b/src/Middleware/Microsoft.AspNetCore.OutputCaching.StackExchangeRedis/test/RedisConnectionFixture.cs @@ -13,8 +13,9 @@ public RedisConnectionFixture() var options = new RedisOutputCacheOptions { Configuration = "127.0.0.1:6379", // TODO: CI test config here - }.GetConfiguredOptions("CI test"); + }.GetConfiguredOptions(); _muxer = ConnectionMultiplexer.Connect(options); + _muxer.AddLibraryNameSuffix("test"); } public IDatabase Database => _muxer.GetDatabase(); diff --git a/src/Servers/HttpSys/src/AsyncAcceptContext.Log.cs b/src/Servers/HttpSys/src/AsyncAcceptContext.Log.cs new file mode 100644 index 000000000000..93d811b6e3ec --- /dev/null +++ b/src/Servers/HttpSys/src/AsyncAcceptContext.Log.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.AspNetCore.Server.HttpSys; + +internal partial class AsyncAcceptContext +{ + private static partial class Log + { + [LoggerMessage(LoggerEventIds.AcceptSetResultFailed, LogLevel.Error, "Error attempting to set 'accept' outcome", EventName = "AcceptSetResultFailed")] + public static partial void AcceptSetResultFailed(ILogger logger, Exception exception); + + // note on "critical": these represent an unexpected IO callback state that needs investigation; see https://github.com/dotnet/aspnetcore/pull/54368/ + + [LoggerMessage(LoggerEventIds.AcceptSetExpectationMismatch, LogLevel.Critical, "Mismatch setting callback expectation - {Value}", EventName = "AcceptSetExpectationMismatch")] + public static partial void AcceptSetExpectationMismatch(ILogger logger, int value); + + [LoggerMessage(LoggerEventIds.AcceptCancelExpectationMismatch, LogLevel.Critical, "Mismatch canceling accept state - {Value}", EventName = "AcceptCancelExpectationMismatch")] + public static partial void AcceptCancelExpectationMismatch(ILogger logger, int value); + + [LoggerMessage(LoggerEventIds.AcceptObserveExpectationMismatch, LogLevel.Critical, "Mismatch observing {Kind} accept callback - {Value}", EventName = "AcceptObserveExpectationMismatch")] + public static partial void AcceptObserveExpectationMismatch(ILogger logger, string kind, int value); + } +} diff --git a/src/Servers/HttpSys/src/AsyncAcceptContext.cs b/src/Servers/HttpSys/src/AsyncAcceptContext.cs index 72131695aed6..a210c19bb4ef 100644 --- a/src/Servers/HttpSys/src/AsyncAcceptContext.cs +++ b/src/Servers/HttpSys/src/AsyncAcceptContext.cs @@ -4,17 +4,23 @@ using System.Diagnostics; using System.Threading.Tasks.Sources; using Microsoft.AspNetCore.HttpSys.Internal; +using Microsoft.Extensions.Logging; namespace Microsoft.AspNetCore.Server.HttpSys; -internal sealed unsafe class AsyncAcceptContext : IValueTaskSource, IDisposable +internal sealed unsafe partial class AsyncAcceptContext : IValueTaskSource, IDisposable { private static readonly IOCompletionCallback IOCallback = IOWaitCallback; private readonly PreAllocatedOverlapped _preallocatedOverlapped; private readonly IRequestContextFactory _requestContextFactory; + private readonly ILogger _logger; + private int _expectedCompletionCount; private NativeOverlapped* _overlapped; + private readonly bool _logExpectationFailures = AppContext.TryGetSwitch( + "Microsoft.AspNetCore.Server.HttpSys.LogAcceptExpectationFailure", out var enabled) && enabled; + // mutable struct; do not make this readonly private ManualResetValueTaskSourceCore _mrvts = new() { @@ -24,11 +30,12 @@ internal sealed unsafe class AsyncAcceptContext : IValueTaskSource AcceptAsync() return new ValueTask(this, _mrvts.Version); } - private void IOCompleted(uint errorCode, uint numBytes) + private void IOCompleted(uint errorCode, uint numBytes, bool managed) { try { + ObserveCompletion(managed); // expectation tracking if (errorCode != UnsafeNclNativeMethods.ErrorCodes.ERROR_SUCCESS && errorCode != UnsafeNclNativeMethods.ErrorCodes.ERROR_MORE_DATA) { - _mrvts.SetException(new HttpSysException((int)errorCode)); - return; + // (keep all the error handling in one place) + throw new HttpSysException((int)errorCode); } Debug.Assert(_requestContext != null); @@ -71,7 +79,14 @@ private void IOCompleted(uint errorCode, uint numBytes) // we want to reuse the acceptContext object for future accepts. _requestContext = null; - _mrvts.SetResult(requestContext); + try + { + _mrvts.SetResult(requestContext); + } + catch (Exception ex) + { + Log.AcceptSetResultFailed(_logger, ex); + } } else { @@ -84,22 +99,69 @@ private void IOCompleted(uint errorCode, uint numBytes) if (statusCode != UnsafeNclNativeMethods.ErrorCodes.ERROR_SUCCESS && statusCode != UnsafeNclNativeMethods.ErrorCodes.ERROR_IO_PENDING) { - // someother bad error, possible(?) return values are: + // some other bad error, possible(?) return values are: // ERROR_INVALID_HANDLE, ERROR_INSUFFICIENT_BUFFER, ERROR_OPERATION_ABORTED - _mrvts.SetException(new HttpSysException((int)statusCode)); + // (keep all the error handling in one place) + throw new HttpSysException((int)statusCode); } } } catch (Exception exception) { - _mrvts.SetException(exception); + try + { + _mrvts.SetException(exception); + } + catch (Exception ex) + { + Log.AcceptSetResultFailed(_logger, ex); + } } } private static unsafe void IOWaitCallback(uint errorCode, uint numBytes, NativeOverlapped* nativeOverlapped) { var acceptContext = (AsyncAcceptContext)ThreadPoolBoundHandle.GetNativeOverlappedState(nativeOverlapped)!; - acceptContext.IOCompleted(errorCode, numBytes); + acceptContext.IOCompleted(errorCode, numBytes, false); + } + + private void SetExpectCompletion() // we anticipate a completion *might* occur + { + // note this is intentionally a "reset and check" rather than Increment, so that we don't spam + // the logs forever if a glitch occurs + var value = Interlocked.Exchange(ref _expectedCompletionCount, 1); // should have been 0 + if (value != 0) + { + if (_logExpectationFailures) + { + Log.AcceptSetExpectationMismatch(_logger, value); + } + Debug.Assert(false, nameof(SetExpectCompletion)); // fail hard in debug + } + } + private void CancelExpectCompletion() // due to error-code etc, we no longer anticipate a completion + { + var value = Interlocked.Decrement(ref _expectedCompletionCount); // should have been 1, so now 0 + if (value != 0) + { + if (_logExpectationFailures) + { + Log.AcceptCancelExpectationMismatch(_logger, value); + } + Debug.Assert(false, nameof(CancelExpectCompletion)); // fail hard in debug + } + } + private void ObserveCompletion(bool managed) // a completion was invoked + { + var value = Interlocked.Decrement(ref _expectedCompletionCount); // should have been 1, so now 0 + if (value != 0) + { + if (_logExpectationFailures) + { + Log.AcceptObserveExpectationMismatch(_logger, managed ? "managed" : "unmanaged", value); + } + Debug.Assert(false, nameof(ObserveCompletion)); // fail hard in debug + } } private uint QueueBeginGetContext() @@ -112,6 +174,7 @@ private uint QueueBeginGetContext() retry = false; uint bytesTransferred = 0; + SetExpectCompletion(); // track this *before*, because of timing vs IOCP (could even be effectively synchronous) statusCode = HttpApi.HttpReceiveHttpRequest( Server.RequestQueue.Handle, _requestContext.RequestId, @@ -123,35 +186,44 @@ private uint QueueBeginGetContext() &bytesTransferred, _overlapped); - if ((statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_CONNECTION_INVALID - || statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_INVALID_PARAMETER) - && _requestContext.RequestId != 0) - { - // ERROR_CONNECTION_INVALID: - // The client reset the connection between the time we got the MORE_DATA error and when we called HttpReceiveHttpRequest - // with the new buffer. We can clear the request id and move on to the next request. - // - // ERROR_INVALID_PARAMETER: Historical check from HttpListener. - // https://referencesource.microsoft.com/#System/net/System/Net/_ListenerAsyncResult.cs,137 - // we might get this if somebody stole our RequestId, - // set RequestId to 0 and start all over again with the buffer we just allocated - // BUGBUG: how can someone steal our request ID? seems really bad and in need of fix. - _requestContext.RequestId = 0; - retry = true; - } - else if (statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_MORE_DATA) - { - // the buffer was not big enough to fit the headers, we need - // to read the RequestId returned, allocate a new buffer of the required size - // (uint)backingBuffer.Length - AlignmentPadding - AllocateNativeRequest(bytesTransferred); - retry = true; - } - else if (statusCode == UnsafeNclNativeMethods.ErrorCodes.ERROR_SUCCESS - && HttpSysListener.SkipIOCPCallbackOnSuccess) + switch (statusCode) { - // IO operation completed synchronously - callback won't be called to signal completion. - IOCompleted(statusCode, bytesTransferred); + case (UnsafeNclNativeMethods.ErrorCodes.ERROR_CONNECTION_INVALID or UnsafeNclNativeMethods.ErrorCodes.ERROR_INVALID_PARAMETER) when _requestContext.RequestId != 0: + // ERROR_CONNECTION_INVALID: + // The client reset the connection between the time we got the MORE_DATA error and when we called HttpReceiveHttpRequest + // with the new buffer. We can clear the request id and move on to the next request. + // + // ERROR_INVALID_PARAMETER: Historical check from HttpListener. + // https://referencesource.microsoft.com/#System/net/System/Net/_ListenerAsyncResult.cs,137 + // we might get this if somebody stole our RequestId, + // set RequestId to 0 and start all over again with the buffer we just allocated + // BUGBUG: how can someone steal our request ID? seems really bad and in need of fix. + CancelExpectCompletion(); + _requestContext.RequestId = 0; + retry = true; + break; + case UnsafeNclNativeMethods.ErrorCodes.ERROR_MORE_DATA: + // the buffer was not big enough to fit the headers, we need + // to read the RequestId returned, allocate a new buffer of the required size + // (uint)backingBuffer.Length - AlignmentPadding + CancelExpectCompletion(); // we'll "expect" again when we retry + AllocateNativeRequest(bytesTransferred); + retry = true; + break; + case UnsafeNclNativeMethods.ErrorCodes.ERROR_SUCCESS: + if (HttpSysListener.SkipIOCPCallbackOnSuccess) + { + // IO operation completed synchronously - callback won't be called to signal completion. + IOCompleted(statusCode, bytesTransferred, true); // marks completion + } + // else: callback fired by IOCP (at some point), which marks completion + break; + case UnsafeNclNativeMethods.ErrorCodes.ERROR_IO_PENDING: + break; // no change to state - callback will occur at some point + default: + // fault code, not expecting an IOCP callback + CancelExpectCompletion(); + break; } } while (retry); diff --git a/src/Servers/HttpSys/src/LoggerEventIds.cs b/src/Servers/HttpSys/src/LoggerEventIds.cs index d550618eab1e..87f8ea56ee9f 100644 --- a/src/Servers/HttpSys/src/LoggerEventIds.cs +++ b/src/Servers/HttpSys/src/LoggerEventIds.cs @@ -54,4 +54,8 @@ internal static class LoggerEventIds public const int RequestValidationFailed = 47; public const int CreateDisconnectTokenError = 48; public const int RequestAborted = 49; + public const int AcceptSetResultFailed = 50; + public const int AcceptSetExpectationMismatch = 51; + public const int AcceptCancelExpectationMismatch = 52; + public const int AcceptObserveExpectationMismatch = 53; } diff --git a/src/Servers/HttpSys/src/MessagePump.cs b/src/Servers/HttpSys/src/MessagePump.cs index ec56271379b2..5cf1c88f0b12 100644 --- a/src/Servers/HttpSys/src/MessagePump.cs +++ b/src/Servers/HttpSys/src/MessagePump.cs @@ -162,7 +162,7 @@ private void ProcessRequestsWorker() Debug.Assert(RequestContextFactory != null); // Allocate and accept context per loop and reuse it for all accepts - var acceptContext = new AsyncAcceptContext(Listener, RequestContextFactory); + var acceptContext = new AsyncAcceptContext(Listener, RequestContextFactory, _logger); var loop = new AcceptLoop(acceptContext, this); diff --git a/src/Servers/HttpSys/test/FunctionalTests/Listener/Utilities.cs b/src/Servers/HttpSys/test/FunctionalTests/Listener/Utilities.cs index 15664bf82119..b7ca7e33df85 100644 --- a/src/Servers/HttpSys/test/FunctionalTests/Listener/Utilities.cs +++ b/src/Servers/HttpSys/test/FunctionalTests/Listener/Utilities.cs @@ -107,7 +107,7 @@ internal static HttpSysListener CreateServerOnExistingQueue(string requestQueueN internal static async Task AcceptAsync(this HttpSysListener server, TimeSpan timeout) { var factory = new TestRequestContextFactory(server); - using var acceptContext = new AsyncAcceptContext(server, factory); + using var acceptContext = new AsyncAcceptContext(server, factory, server.Logger); async Task AcceptAsync() { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs index f90f728a34fe..3006288f4b8c 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2FrameWriter.cs @@ -370,19 +370,36 @@ public void UpdateMaxFrameSize(uint maxFrameSize) } } + /// + /// Call while in the . + /// + /// true if already completed. + private bool CompleteUnsynchronized() + { + if (_completed) + { + return true; + } + + _completed = true; + _outputWriter.Abort(); + + return false; + } + public void Complete() { lock (_writeLock) { - if (_completed) + if (CompleteUnsynchronized()) { return; } - - _completed = true; - AbortConnectionFlowControl(); - _outputWriter.Abort(); } + + // Call outside of _writeLock as this can call Http2OutputProducer.Stop which can acquire Http2OutputProducer._dataWriterLock + // which is not the desired lock order + AbortConnectionFlowControl(); } public Task ShutdownAsync() @@ -404,8 +421,15 @@ public void Abort(ConnectionAbortedException error) _aborted = true; _connectionContext.Abort(error); - Complete(); + if (CompleteUnsynchronized()) + { + return; + } } + + // Call outside of _writeLock as this can call Http2OutputProducer.Stop which can acquire Http2OutputProducer._dataWriterLock + // which is not the desired lock order + AbortConnectionFlowControl(); } private ValueTask FlushEndOfStreamHeadersAsync(Http2Stream stream) @@ -478,7 +502,7 @@ private void WriteResponseHeadersUnsynchronized(int streamId, int statusCode, Ht _outgoingFrame.PrepareHeaders(headerFrameFlags, streamId); var buffer = _headerEncodingBuffer.AsSpan(); var done = HPackHeaderWriter.BeginEncodeHeaders(statusCode, _hpackEncoder, _headersEnumerator, buffer, out var payloadLength); - FinishWritingHeaders(streamId, payloadLength, done); + FinishWritingHeadersUnsynchronized(streamId, payloadLength, done); } // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. // Since we allow custom header encoders we don't know what type of exceptions to expect. @@ -519,7 +543,7 @@ private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in _outgoingFrame.PrepareHeaders(Http2HeadersFrameFlags.END_STREAM, streamId); var buffer = _headerEncodingBuffer.AsSpan(); var done = HPackHeaderWriter.BeginEncodeHeaders(_hpackEncoder, _headersEnumerator, buffer, out var payloadLength); - FinishWritingHeaders(streamId, payloadLength, done); + FinishWritingHeadersUnsynchronized(streamId, payloadLength, done); } // Any exception from the HPack encoder can leave the dynamic table in a corrupt state. // Since we allow custom header encoders we don't know what type of exceptions to expect. @@ -533,7 +557,7 @@ private ValueTask WriteDataAndTrailersAsync(Http2Stream stream, in } } - private void FinishWritingHeaders(int streamId, int payloadLength, bool done) + private void FinishWritingHeadersUnsynchronized(int streamId, int payloadLength, bool done) { var buffer = _headerEncodingBuffer.AsSpan(); _outgoingFrame.PayloadLength = payloadLength; @@ -925,6 +949,11 @@ private void ConsumeConnectionWindow(long bytes) } } + /// + /// Do not call this method under the _writeLock. + /// This method can call Http2OutputProducer.Stop which can acquire Http2OutputProducer._dataWriterLock + /// which is not the desired lock order + /// private void AbortConnectionFlowControl() { lock (_windowUpdateLock) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs index 8f13acbc2763..f65890adf7cb 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2OutputProducer.cs @@ -590,6 +590,7 @@ public void Reset() internal void OnRequestProcessingEnded() { + var shouldCompleteStream = false; lock (_dataWriterLock) { if (_requestProcessingComplete) @@ -600,15 +601,24 @@ internal void OnRequestProcessingEnded() _requestProcessingComplete = true; - if (_completedResponse) - { - Stream.CompleteStream(errored: false); - } + shouldCompleteStream = _completedResponse; + } + + // Complete outside of lock, anything this method does that needs a lock will acquire a lock itself. + // Additionally, this method should only be called once per Reset so calling outside of the lock is fine from the perspective + // of multiple threads calling OnRequestProcessingEnded. + if (shouldCompleteStream) + { + Stream.CompleteStream(errored: false); } + } internal ValueTask CompleteResponseAsync() { + var shouldCompleteStream = false; + ValueTask task = default; + lock (_dataWriterLock) { if (_completedResponse) @@ -619,8 +629,6 @@ internal ValueTask CompleteResponseAsync() _completedResponse = true; - ValueTask task = default; - if (_resetErrorCode is { } error) { // If we have an error code to write, write it now that we're done with the response. @@ -628,13 +636,18 @@ internal ValueTask CompleteResponseAsync() task = _frameWriter.WriteRstStreamAsync(StreamId, error); } - if (_requestProcessingComplete) - { - Stream.CompleteStream(errored: false); - } + shouldCompleteStream = _requestProcessingComplete; + } - return task; + // Complete outside of lock, anything this method does that needs a lock will acquire a lock itself. + // CompleteResponseAsync also should never be called in parallel so calling this outside of the lock doesn't + // cause any weirdness with parallel threads calling this method and no longer waiting on the stream completion call. + if (shouldCompleteStream) + { + Stream.CompleteStream(errored: false); } + + return task; } internal Memory GetFakeMemory(int minSize) diff --git a/src/SignalR/server/StackExchangeRedis/src/RedisHubLifetimeManager.cs b/src/SignalR/server/StackExchangeRedis/src/RedisHubLifetimeManager.cs index acd18262d16e..f1743fa5f2d6 100644 --- a/src/SignalR/server/StackExchangeRedis/src/RedisHubLifetimeManager.cs +++ b/src/SignalR/server/StackExchangeRedis/src/RedisHubLifetimeManager.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StackExchange.Redis; +using RedisProtocol = Microsoft.AspNetCore.SignalR.StackExchangeRedis.Internal.RedisProtocol; // to disambiguate from StackExchange.Redis.RedisProtocol namespace Microsoft.AspNetCore.SignalR.StackExchangeRedis; diff --git a/src/SignalR/server/StackExchangeRedis/test/TestConnectionMultiplexer.cs b/src/SignalR/server/StackExchangeRedis/test/TestConnectionMultiplexer.cs index 953d1d46a83e..1768a7f97471 100644 --- a/src/SignalR/server/StackExchangeRedis/test/TestConnectionMultiplexer.cs +++ b/src/SignalR/server/StackExchangeRedis/test/TestConnectionMultiplexer.cs @@ -1,19 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Net; using System.Reflection; -using System.Threading; -using System.Threading.Tasks; using StackExchange.Redis; using StackExchange.Redis.Maintenance; using StackExchange.Redis.Profiling; -using Xunit; namespace Microsoft.AspNetCore.SignalR.Tests; @@ -237,6 +230,8 @@ public IServer[] GetServers() } public ValueTask DisposeAsync() => default; + + public void AddLibraryNameSuffix(string suffix) { } // don't need to implement } public class TestRedisServer diff --git a/src/submodules/googletest b/src/submodules/googletest index 48729681ad88..77afe8e0149c 160000 --- a/src/submodules/googletest +++ b/src/submodules/googletest @@ -1 +1 @@ -Subproject commit 48729681ad88a89061344ee541b4548833077e00 +Subproject commit 77afe8e0149c207edd9561c28de6d2226673b51f pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy