Update e2e tests and e2e test infrastructure (#414)

This commit is contained in:
Ivan 2023-04-20 14:42:27 +02:00 committed by GitHub
parent 920b830bd1
commit fc8786b149
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 167 additions and 170 deletions

View File

@ -33,7 +33,7 @@ jobs:
3.0.x 3.0.x
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 2.2.402 3.1.404 '3.0' run: __tests__/verify-dotnet.ps1 -Patterns "^2.2.402$", "^3.1.404$", "^3.0"
test-setup-full-version: test-setup-full-version:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -60,13 +60,9 @@ jobs:
source-url: https://api.nuget.org/v3/index.json source-url: https://api.nuget.org/v3/index.json
env: env:
NUGET_AUTH_TOKEN: NOTATOKEN NUGET_AUTH_TOKEN: NOTATOKEN
- name: Verify nuget config file
shell: pwsh
run: |
if (-Not (Test-Path "../nuget.config")) { throw "nuget file not generated correctly" }
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 3.1.201 2.2.402 run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$", "^2.2.402$" -CheckNugetConfig
test-setup-without-patch-version: test-setup-without-patch-version:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -91,7 +87,7 @@ jobs:
dotnet-version: '2.2' dotnet-version: '2.2'
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 3.1 2.2 run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2"
test-setup-prerelease-version: test-setup-prerelease-version:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -105,17 +101,13 @@ jobs:
- name: Clear toolcache - name: Clear toolcache
shell: pwsh shell: pwsh
run: __tests__/clear-toolcache.ps1 ${{ runner.os }} run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
- name: Setup dotnet '2.2'
uses: ./
with:
dotnet-version: '2.2'
- name: Setup dotnet '3.1.100-preview1-014459' - name: Setup dotnet '3.1.100-preview1-014459'
uses: ./ uses: ./
with: with:
dotnet-version: '3.1.100-preview1-014459' dotnet-version: '3.1.100-preview1-014459'
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 3.1.100-preview1-014459 run: __tests__/verify-dotnet.ps1 -Patterns "3.1.100-preview1-014459"
test-setup-latest-patch-version: test-setup-latest-patch-version:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -139,7 +131,7 @@ jobs:
dotnet-version: 2.2.X dotnet-version: 2.2.X
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 '2.2' '3.1' run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1"
test-setup-with-wildcard: test-setup-with-wildcard:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -163,7 +155,7 @@ jobs:
dotnet-version: 2.2.* dotnet-version: 2.2.*
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 3.1 2.2 run: __tests__/verify-dotnet.ps1 -Patterns "^3.1", "^2.2"
test-setup-global-json-specified-and-version: test-setup-global-json-specified-and-version:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -181,7 +173,7 @@ jobs:
shell: bash shell: bash
run: | run: |
mkdir subdirectory mkdir subdirectory
echo '{"sdk":{"version": "2.2","rollForward": "latestFeature"}}' > ./subdirectory/global.json echo '{"sdk":{"version": "2.2.207","rollForward": "latestFeature"}}' > ./subdirectory/global.json
- name: Setup dotnet - name: Setup dotnet
uses: ./ uses: ./
with: with:
@ -189,7 +181,7 @@ jobs:
global-json-file: ./subdirectory/global.json global-json-file: ./subdirectory/global.json
- name: Verify dotnet - name: Verify dotnet
shell: pwsh shell: pwsh
run: __tests__/verify-dotnet.ps1 2.2 3.1 run: __tests__/verify-dotnet.ps1 -Patterns "^2.2", "^3.1"
test-setup-with-dotnet-quality: test-setup-with-dotnet-quality:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -209,12 +201,9 @@ jobs:
with: with:
dotnet-version: '7.0' dotnet-version: '7.0'
dotnet-quality: 'preview' dotnet-quality: 'preview'
- name: Verify preview version - name: Verify dotnet
shell: pwsh shell: pwsh
run: | run: __tests__/verify-dotnet.ps1 -Patterns "^7\.0\.\d+-"
$version = & dotnet --version
Write-Host "Installed version: $version"
if (-not ($version.Contains("preview") -or $version.Contains("rc"))) { throw "Unexpected version" }
test-dotnet-version-output-during-single-version-installation: test-dotnet-version-output-during-single-version-installation:
runs-on: ${{ matrix.operating-system }} runs-on: ${{ matrix.operating-system }}
@ -273,7 +262,7 @@ jobs:
test-proxy: test-proxy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: container:
image: mcr.microsoft.com/dotnet/core/runtime-deps:3.0-bionic image: ubuntu:latest
options: --dns 127.0.0.1 options: --dns 127.0.0.1
services: services:
squid-proxy: squid-proxy:
@ -286,21 +275,29 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
- name: Clear tool cache - name: Install Powershell
run: rm -rf "/usr/share/dotnet"
- name: Install curl
run: | run: |
apt update apt-get update
apt -y install curl apt-get install -y wget apt-transport-https software-properties-common
- name: Setup dotnet 3.1.201 wget -q "https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb"
dpkg -i packages-microsoft-prod.deb
rm packages-microsoft-prod.deb
apt-get update
apt-get install -y powershell
- name: Clear toolcache
shell: pwsh
run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
- name: Setup dotnet 6.0
uses: ./ uses: ./
with: with:
dotnet-version: 3.1.201 dotnet-version: 6.0
source-url: https://api.nuget.org/v3/index.json source-url: https://api.nuget.org/v3/index.json
env: env:
NUGET_AUTH_TOKEN: NOTATOKEN NUGET_AUTH_TOKEN: NOTATOKEN
- name: Verify dotnet - name: Verify dotnet
run: __tests__/verify-dotnet.sh 3.1.201 shell: pwsh
run: |
__tests__/verify-dotnet.ps1 -Patterns "^6.0" -CheckNugetConfig
test-bypass-proxy: test-bypass-proxy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -311,7 +308,8 @@ jobs:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
- name: Clear toolcache - name: Clear toolcache
run: rm -rf "/usr/share/dotnet" shell: pwsh
run: __tests__/clear-toolcache.ps1 ${{ runner.os }}
- name: Setup dotnet 3.1.201 - name: Setup dotnet 3.1.201
uses: ./ uses: ./
with: with:
@ -320,4 +318,5 @@ jobs:
env: env:
NUGET_AUTH_TOKEN: NOTATOKEN NUGET_AUTH_TOKEN: NOTATOKEN
- name: Verify dotnet - name: Verify dotnet
run: __tests__/verify-dotnet.sh 3.1.201 shell: pwsh
run: __tests__/verify-dotnet.ps1 -Patterns "^3.1.201$" -CheckNugetConfig

View File

@ -18,7 +18,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
operating-system: [ubuntu-latest, windows-latest, macOS-latest] operating-system: [ubuntu-latest, windows-latest, macOS-latest]
dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0'] dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0', '6.0', '7.0', '8.0']
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v3
@ -29,9 +29,7 @@ jobs:
uses: ./ uses: ./
with: with:
dotnet-version: ${{ matrix.dotnet-version }} dotnet-version: ${{ matrix.dotnet-version }}
- name: Check installed version - name: Verify installed version
shell: pwsh shell: pwsh
run: | run: |
$version = & dotnet --version __tests__/verify-dotnet.ps1 -Patterns "^${{ matrix.dotnet-version }}"
Write-Host "Installed version: $version"
if (-not $version.StartsWith("${{ matrix.dotnet-version }}")) { throw "Unexpected version" }

4
.gitignore vendored
View File

@ -3,8 +3,8 @@ global.json
lib/ lib/
node_modules/ node_modules/
__tests__/runner/* __tests__/runner/*
__tests__/sample-csproj/bin/ __tests__/e2e-test-csproj/bin/
__tests__/sample-csproj/obj/ __tests__/e2e-test-csproj/obj/
# Rest of the file pulled from https://github.com/github/gitignore/blob/master/Node.gitignore # Rest of the file pulled from https://github.com/github/gitignore/blob/master/Node.gitignore
# Logs # Logs

View File

@ -0,0 +1,18 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
namespace test_csproj
{
[TestClass]
public class Test
{
[TestMethod]
public void TestMethod()
{
Console.WriteLine("TestMethod");
int calculatedResult = 1000 / 25;
int expectedResult = 40;
Assert.AreEqual(calculatedResult, expectedResult);
}
}
}

View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(TEST_TARGET_FRAMEWORK)</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<!-- These packages will be downloaded over the network for testing proxy settings -->
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0-preview-20170810-02" />
<PackageReference Include="MSTest.TestAdapter" Version="1.1.18" />
<PackageReference Include="MSTest.TestFramework" Version="1.1.18" />
</ItemGroup>
</Project>

View File

@ -1,15 +0,0 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
namespace sample_csproj
{
[TestClass]
public class Program
{
[TestMethod]
public void TestMethod1()
{
Console.WriteLine("Hello, World!");
}
}
}

View File

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>netcoreapp3.1;netcoreapp3.0;netcoreapp2.2</TargetFrameworks>
<RootNamespace>sample_csproj</RootNamespace>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<!-- These packages will be downloaded over the network for testing proxy settings -->
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
</ItemGroup>
</Project>

View File

@ -1,73 +1,117 @@
if (!$args[0]) <#
{ .DESCRIPTION
throw "Must supply dotnet version argument" Verifies that installed on the machine .NET SDK versions match the input patterns.
Optionally checks that the nuget.config file is generated correctly.
.PARAMETER Patterns
Specifies the regular expression patterns that should be matched with the installed
on the machine .NET SDK versions. The number of patterns should be equal to the number
of installed .NET versions.
.PARAMETER CheckNugetConfig
Switches the check for the existence of the nuget.config file.
.EXAMPLE
PS> .\verify-dotnet.ps1 -Paterns "^3.1.200$", "^6.0" -CheckNugetConfig
#>
param(
[ValidateNotNullOrEmpty()]
[Parameter(Mandatory=$true)]
[string[]]$Patterns,
[switch]$CheckNugetConfig
)
$PatternsList = [System.Collections.ArrayList]($Patterns)
if ($CheckNugetConfig.IsPresent -and !(Test-Path "../nuget.config")) {
throw "The nuget.config file is not generated correctly."
} }
Write-Host "These patterns were supplied to the script: $($PatternsList -join ', ')."
$dotnet = Get-Command dotnet | Select-Object -First 1 | ForEach-Object { $_.Path } $dotnet = Get-Command dotnet | Select-Object -First 1 | ForEach-Object { $_.Path }
Write-Host "Found '$dotnet'" Write-Host "Found: '$dotnet'"
if($args.count -eq 1)
{
$version = & $dotnet --version | Out-String | ForEach-Object { $_.Trim() }
Write-Host "Version $version"
if (-not ($version.StartsWith($args[0].ToString())))
{
Write-Host "PATH='$env:PATH'"
throw "Unexpected version"
}
}
if ($args[1])
{
# SDKs are listed on multiple lines with the path afterwards in square brackets # SDKs are listed on multiple lines with the path afterwards in square brackets
$versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() } $Versions = & $dotnet --list-sdks | ForEach-Object { $_.SubString(0, $_.IndexOf('[')).Trim() }
Write-Host "Installed versions: $versions" Write-Host "Found installed versions: $($Versions -join ', ')."
$InstalledVersionCount = 0 $InstalledVersionCount = $Versions.Count
foreach($arg in $args){
foreach ($version in $versions) foreach($version in $Versions)
{ {
if ($version.StartsWith($arg.ToString())) foreach($pattern in $PatternsList)
{ {
$InstalledVersionCount++ if ($version -match $pattern)
}
}
}
if ( $InstalledVersionCount -ne $args.Count)
{ {
Write-Host "PATH='$env:PATH'" $PatternsList.Remove($pattern)
throw "Unexpected version" $InstalledVersionCount--
break
}
} }
} }
Write-Host "Building sample csproj" if ( $InstalledVersionCount -ne 0)
& $dotnet build __tests__/sample-csproj/ --no-cache {
throw "An unexpected version of Dotnet is found on the machine, please check the correctness of the -Patterns input."
}
$workingDir = Get-Location
$testProjectDir = "./__tests__/e2e-test-csproj"
Write-Host "Changing directory to the $testProjectDir"
Set-Location $testProjectDir
$targetFrameworkVersionMap = @{
"1.0" = "netcoreapp1.0";
"1.1" = "netcoreapp1.1";
"2.0" = "netcoreapp2.0";
"2.1" = "netcoreapp2.1";
"2.2" = "netcoreapp2.2";
"3.0" = "netcoreapp3.0";
"3.1" = "netcoreapp3.1";
"5.0" = "net5.0";
"6.0" = "net6.0";
"7.0" = "net7.0";
"8.0" = "net8.0";
}
foreach ($version in $Versions)
{
# Creating temporary global.json file inside e2e-test-csproj dir and setting exact version of .NET inside allows to override default behavior of .NET and run build and tests on that exact version.
Write-Host "Creating temporary global.json file for $version .NET version."
& $dotnet new globaljson --sdk-version $version --force | Out-Null
if (!(Test-Path "./global.json"))
{
throw "An error occured while creating the global.json file. Exit code: $LASTEXITCODE"
}
Write-Host "The global.json file for the version $version is created. Currently used .NET version is: $(& $dotnet --version)."
# Environment variable TEST_TARGET_FRAMEWORK is used inside the test.csproj file to target required framework version
$version -match "^(?<key>\d+\.\d+)" | Out-Null
if (!($targetFrameworkVersionMap.ContainsKey($Matches.key)))
{
throw "The map with the framework targets doesn't contain a target name for the version $version."
}
Write-Host "Setting the TEST_TARGET_FRAMEWORK environment variable to $($targetFrameworkVersionMap[$Matches.key])"
[Environment]::SetEnvironmentVariable('TEST_TARGET_FRAMEWORK', $($targetFrameworkVersionMap[$Matches.key]))
Write-Host "Building test C# project with $version .NET version."
& $dotnet build --no-cache
if ($LASTEXITCODE -ne 0) if ($LASTEXITCODE -ne 0)
{ {
throw "Unexpected exit code $LASTEXITCODE" throw "Building process is not successful, exit code: $LASTEXITCODE"
} }
Write-Host "Testing compiled app" Write-Host "Testing compiled C# project with $version .NET version."
$sample_output = "$(dotnet test __tests__/sample-csproj/ --no-build)" & $dotnet test --no-build
Write-Host "Sample output: $sample_output" if ($LASTEXITCODE -ne 0)
# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once
if ($args[1])
{ {
if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*") throw "Testing process is not successful, exit code: $LASTEXITCODE"
{
throw "Unexpected output"
}
}
if ($args[2])
{
if ($sample_output -notlike "*Test Run Successful.*Test Run Successful.*Test Run Successful.*")
{
throw "Unexpected output"
}
}
else
{
if ($sample_output -notlike "*Test Run Successful.*")
{
throw "Unexpected output"
} }
Write-Host "Tests are completed successfully!"
Write-Host "Removing temporary global.json file."
Remove-Item ./global.json
} }
Set-Location $workingDir

View File

@ -1,44 +0,0 @@
if [ -z "$1" ]; then
echo "Must supply dotnet version argument"
exit 1
fi
if [ ! -f "../nuget.config" ]; then
echo "nuget file not generated correctly"
exit 1
fi
dotnet_version="$(dotnet --version)"
echo "Found dotnet version '$dotnet_version'"
if [ -z "$(echo $dotnet_version | grep $1)" ]; then
echo "Unexpected version"
exit 1
fi
if [ -n "$2" ]; then
dotnet_version="$(dotnet --list-sdks)"
echo "Found dotnet version '$dotnet_version'"
if [ -z "$(echo $dotnet_version | grep $2)" ]; then
echo "Unexpected version"
exit 1
fi
fi
echo "Building sample csproj"
dotnet build __tests__/sample-csproj/ --no-cache || exit 1
echo "Testing compiled app"
sample_output=$(dotnet test __tests__/sample-csproj/ --no-build)
echo "Sample output: $sample_output"
# For Side-by-Side installs we want to run the tests twice, for a single install the tests will run once
if [ -n "$2" ]; then
if [ -z "$(echo $sample_output | grep "Test Run Successful.*Test Run Successful.")" ]; then
echo "Unexpected output"
exit 1
fi
else
if [ -z "$(echo $sample_output | grep "Test Run Successful.")" ]; then
echo "Unexpected output"
exit 1
fi
fi

View File

@ -67,7 +67,7 @@ Pull requests are the easiest way to contribute changes to git repos at GitHub.
**Learn more about how to implement tests:** **Learn more about how to implement tests:**
Adding or changing tests is an integral part of making a change to the code. Adding or changing tests is an integral part of making a change to the code.
Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [workflow.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/workflow.yml). Unit tests are in the `__tests__` folder, and end-to-end tests are in the `workflows` folder, particularly in the [e2e-tests.yml](https://github.com/actions/setup-dotnet/blob/main/.github/workflows/e2e-tests.yml).
- The contributor can add various types of tests (like unit tests or end-to-end tests), which, in his opinion, will be necessary and sufficient for testing new or changed functionality - The contributor can add various types of tests (like unit tests or end-to-end tests), which, in his opinion, will be necessary and sufficient for testing new or changed functionality
- Tests should cover a successful execution, as well as some edge cases and possible errors - Tests should cover a successful execution, as well as some edge cases and possible errors