V3 - Use new .NET CDN URLs and update to latest install scripts (#567)

* Update new cdn

* new cdn changes

* update latest install scripts

* Format install scripts

* Fix Ubuntu-latest issues

* Fallback logic

* Format check

* Update signed version

.github/workflows/e2e-tests.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -40,7 +40,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macos-13]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -94,7 +94,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macos-13]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -114,7 +114,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -138,7 +138,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -159,7 +159,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -183,7 +183,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -209,7 +209,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -234,7 +234,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -256,7 +256,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macos-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macos-latest]
     env:
       NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
     steps:
@@ -287,7 +287,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macos-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macos-latest]
     env:
       NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
     steps:
@@ -316,7 +316,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -342,7 +342,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -366,7 +366,7 @@ jobs:
           if (-not ($version -eq '${{steps.step2.outputs.dotnet-version}}')) { throw "Unexpected output value" }
 
   test-proxy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     container:
       image: ubuntu:latest
       options: --dns 127.0.0.1
@@ -406,7 +406,7 @@ jobs:
           __tests__/verify-dotnet.ps1 -Patterns "^6.0" -CheckNugetConfig
 
   test-bypass-proxy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       https_proxy: http://no-such-proxy:3128
       no_proxy: github.com,dotnetcli.blob.core.windows.net,download.visualstudio.microsoft.com,api.nuget.org,dotnetcli.azureedge.net

.github/workflows/test-dotnet.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+        operating-system: [ubuntu-22.04, windows-latest, macOS-latest]
         dotnet-version: ['2.1', '2.2', '3.0', '3.1', '5.0', '6.0', '7.0', '8.0']
     steps:
       - name: Checkout

dist/setup/index.js

@@ -71264,7 +71264,13 @@ class DotnetVersionResolver {
                 allowRetries: true,
                 maxRetries: 3
             });
-            const response = yield httpClient.getJson(DotnetVersionResolver.DotNetCoreIndexUrl);
+            let response;
+            try {
+                response = yield httpClient.getJson(DotnetVersionResolver.DotNetCoreIndexUrl);
+            }
+            catch (error) {
+                response = yield httpClient.getJson(DotnetVersionResolver.DotnetCoreIndexFallbackUrl);
+            }
             const result = response.result || {};
             const releasesInfo = result['releases-index'];
             const releaseInfo = releasesInfo.find(info => {
@@ -71279,7 +71285,8 @@ class DotnetVersionResolver {
     }
 }
 exports.DotnetVersionResolver = DotnetVersionResolver;
-DotnetVersionResolver.DotNetCoreIndexUrl = 'https://dotnetcli.azureedge.net/dotnet/release-metadata/releases-index.json';
+DotnetVersionResolver.DotNetCoreIndexUrl = 'https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json';
+DotnetVersionResolver.DotnetCoreIndexFallbackUrl = 'https://dotnetcli.azureedge.net/dotnet/release-metadata/releases-index.json';
 class DotnetCoreInstaller {
     constructor(version, quality) {
         this.version = version;

externals/install-dotnet.sh

@@ -298,11 +298,20 @@ get_machine_architecture() {
     if command -v uname > /dev/null; then
         CPUName=$(uname -m)
         case $CPUName in
+        armv1*|armv2*|armv3*|armv4*|armv5*|armv6*)
+            echo "armv6-or-below"
+            return 0
+            ;;
         armv*l)
             echo "arm"
             return 0
             ;;
         aarch64|arm64)
+            if [ "$(getconf LONG_BIT)" -lt 64 ]; then
+                # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS)
+                echo "arm"
+                return 0
+            fi
             echo "arm64"
             return 0
             ;;
@@ -310,6 +319,22 @@ get_machine_architecture() {
             echo "s390x"
             return 0
             ;;
+        ppc64le)
+            echo "ppc64le"
+            return 0
+            ;;
+        loongarch64)
+            echo "loongarch64"
+            return 0
+            ;;
+        riscv64)
+            echo "riscv64"
+            return 0
+            ;;
+        powerpc|ppc)
+            echo "ppc"
+            return 0
+            ;;
         esac
     fi
 
@@ -326,7 +351,13 @@ get_normalized_architecture_from_architecture() {
     local architecture="$(to_lowercase "$1")"
 
     if [[ $architecture == \<auto\> ]]; then
-        echo "$(get_machine_architecture)"
+        machine_architecture="$(get_machine_architecture)"
+        if [[ "$machine_architecture" == "armv6-or-below" ]]; then
+            say_err "Architecture \`$machine_architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues"
+            return 1
+        fi
+
+        echo $machine_architecture
         return 0
     fi
 
@@ -347,6 +378,14 @@ get_normalized_architecture_from_architecture() {
             echo "s390x"
             return 0
             ;;
+        ppc64le)
+            echo "ppc64le"
+            return 0
+            ;;
+        loongarch64)
+            echo "loongarch64"
+            return 0
+            ;;
     esac
 
     say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues"
@@ -384,11 +423,17 @@ get_normalized_architecture_for_specific_sdk_version() {
 # args:
 # version or channel - $1
 is_arm64_supported() {
-    #any channel or version that starts with the specified versions
+    # Extract the major version by splitting on the dot
-    case "$1" in
+    major_version="${1%%.*}"
-        ( "1"* | "2"* | "3"*  | "4"* | "5"*) 
+
+    # Check if the major version is a valid number and less than 6
+    case "$major_version" in
+        [0-9]*)  
+            if [ "$major_version" -lt 6 ]; then
                 echo false
                 return 0
+            fi
+            ;;
     esac
 
     echo true
@@ -407,8 +452,13 @@ get_normalized_os() {
                 echo "$osname"
                 return 0
                 ;;
+            macos)
+                osname='osx'
+                echo "$osname"
+                return 0
+                ;;
             *)
-                say_err "'$user_defined_os' is not a supported value for --os option, supported values are: osx, linux, linux-musl, freebsd, rhel.6. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues."
+                say_err "'$user_defined_os' is not a supported value for --os option, supported values are: osx, macos, linux, linux-musl, freebsd, rhel.6. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues."
                 return 1
                 ;;
         esac
@@ -538,6 +588,40 @@ is_dotnet_package_installed() {
     fi
 }
 
+# args:
+# downloaded file - $1
+# remote_file_size - $2
+validate_remote_local_file_sizes() 
+{
+    eval $invocation
+
+    local downloaded_file="$1"
+    local remote_file_size="$2"
+    local file_size=''
+
+    if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+        file_size="$(stat -c '%s' "$downloaded_file")"
+    elif [[ "$OSTYPE" == "darwin"* ]]; then
+        # hardcode in order to avoid conflicts with GNU stat
+        file_size="$(/usr/bin/stat -f '%z' "$downloaded_file")"
+    fi  
+    
+    if [ -n "$file_size" ]; then
+        say "Downloaded file size is $file_size bytes."
+
+        if [ -n "$remote_file_size" ] && [ -n "$file_size" ]; then
+            if [ "$remote_file_size" -ne "$file_size" ]; then
+                say "The remote and local file sizes are not equal. The remote file size is $remote_file_size bytes and the local size is $file_size bytes. The local package may be corrupted."
+            else
+                say "The remote and local file sizes are equal."
+            fi
+        fi
+        
+    else
+        say "Either downloaded or local package size can not be measured. One of them may be corrupted."      
+    fi 
+}
+
 # args:
 # azure_feed - $1
 # channel - $2
@@ -872,6 +956,37 @@ get_absolute_path() {
     return 0
 }
 
+# args:
+# override - $1 (boolean, true or false)
+get_cp_options() {
+    eval $invocation
+
+    local override="$1"
+    local override_switch=""
+
+    if [ "$override" = false ]; then
+        override_switch="-n"
+
+        # create temporary files to check if 'cp -u' is supported
+        tmp_dir="$(mktemp -d)"
+        tmp_file="$tmp_dir/testfile"
+        tmp_file2="$tmp_dir/testfile2"
+
+        touch "$tmp_file"
+
+        # use -u instead of -n if it's available
+        if cp -u "$tmp_file" "$tmp_file2" 2>/dev/null; then
+            override_switch="-u"
+        fi
+
+        # clean up
+        rm -f "$tmp_file" "$tmp_file2"
+        rm -rf "$tmp_dir"
+    fi
+
+    echo "$override_switch"
+}
+
 # args:
 # input_files - stdin
 # root_path - $1
@@ -883,15 +998,7 @@ copy_files_or_dirs_from_list() {
     local root_path="$(remove_trailing_slash "$1")"
     local out_path="$(remove_trailing_slash "$2")"
     local override="$3"
-    local osname="$(get_current_os_name)"
+    local override_switch="$(get_cp_options "$override")"
-    local override_switch=$(
-        if [ "$override" = false ]; then
-            if [ "$osname" = "linux-musl" ]; then
-                printf -- "-u";
-            else
-                printf -- "-n";
-            fi
-        fi)
 
     cat | uniq | while read -r file_path; do
         local path="$(remove_beginning_slash "${file_path#$root_path}")"
@@ -906,14 +1013,39 @@ copy_files_or_dirs_from_list() {
     done
 }
 
+# args:
+# zip_uri - $1
+get_remote_file_size() {
+    local zip_uri="$1"
+
+    if machine_has "curl"; then
+        file_size=$(curl -sI  "$zip_uri" | grep -i content-length | awk '{ num = $2 + 0; print num }')
+    elif machine_has "wget"; then
+        file_size=$(wget --spider --server-response -O /dev/null "$zip_uri" 2>&1 | grep -i 'Content-Length:' | awk '{ num = $2 + 0; print num }')
+    else
+        say "Neither curl nor wget is available on this system."
+        return
+    fi
+
+    if [ -n "$file_size" ]; then
+        say "Remote file $zip_uri size is $file_size bytes."
+        echo "$file_size"
+    else
+        say_verbose "Content-Length header was not extracted for $zip_uri."
+        echo ""
+    fi
+}
+
 # args:
 # zip_path - $1
 # out_path - $2
+# remote_file_size - $3
 extract_dotnet_package() {
     eval $invocation
 
     local zip_path="$1"
     local out_path="$2"
+    local remote_file_size="$3"
 
     local temp_out_path="$(mktemp -d "$temporary_file_template")"
 
@@ -924,8 +1056,12 @@ extract_dotnet_package() {
     find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false
     find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files"
     
+    validate_remote_local_file_sizes "$zip_path" "$remote_file_size"
+    
     rm -rf "$temp_out_path"
-    rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed"
+    if [ -z ${keep_zip+x} ]; then
+        rm -f "$zip_path" && say_verbose "Temporary archive file $zip_path was removed"
+    fi
 
     if [ "$failed" = true ]; then
         say_err "Extraction failed"
@@ -1136,6 +1272,61 @@ downloadwget() {
     return 0
 }
 
+extract_stem() {
+    local url="$1"
+    # extract the protocol
+    proto="$(echo $1 | grep :// | sed -e's,^\(.*://\).*,\1,g')"
+    # remove the protocol
+    url="${1/$proto/}"
+    # extract the path (if any) - since we know all of our feeds have a first path segment, we can skip the first one. otherwise we'd use -f2- to get the full path
+    full_path="$(echo $url | grep / | cut -d/ -f2-)"
+    path="$(echo $full_path | cut -d/ -f2-)"
+    echo $path
+}
+
+check_url_exists() {
+    eval $invocation
+    local url="$1"
+
+    local code=""
+    if machine_has "curl"
+    then
+        code=$(curl --head -o /dev/null -w "%{http_code}" -s --fail "$url");
+    elif machine_has "wget"
+    then
+        # get the http response, grab the status code
+        server_response=$(wget -qO- --method=HEAD --server-response "$url" 2>&1)
+        code=$(echo "$server_response" | grep "HTTP/" | awk '{print $2}')
+    fi
+    if [ $code = "200" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+sanitize_redirect_url() {
+    eval $invocation
+
+    local url_stem
+    url_stem=$(extract_stem "$1")
+    say_verbose "Checking configured feeds for the asset at ${yellow:-}$url_stem${normal:-}"
+
+    for feed in "${feeds[@]}"
+    do
+        local trial_url="$feed/$url_stem"
+        say_verbose "Checking ${yellow:-}$trial_url${normal:-}"
+        if check_url_exists "$trial_url"; then
+            say_verbose "Found a match at ${yellow:-}$trial_url${normal:-}"
+            echo "$trial_url"
+            return 0
+        else
+            say_verbose "No match at ${yellow:-}$trial_url${normal:-}"
+        fi
+    done
+    return 1
+}
+
 get_download_link_from_aka_ms() {
     eval $invocation
 
@@ -1172,6 +1363,12 @@ get_download_link_from_aka_ms() {
     http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' )
     # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404).
     broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' )
+    # The response may end without final code 2xx/4xx/5xx somehow, e.g. network restrictions on www.bing.com causes redirecting to bing.com fails with connection refused.
+    # In this case it should not exclude the last.
+    last_http_code=$(  echo "$http_codes" | tail -n 1 )
+    if ! [[ $last_http_code =~ ^(2|4|5)[0-9][0-9]$ ]]; then
+        broken_redirects=$( echo "$http_codes" | grep -v '301' )
+    fi
 
     # All HTTP codes are 301 (Moved Permanently), the redirect link exists.
     if [[ -z "$broken_redirects" ]]; then
@@ -1182,6 +1379,11 @@ get_download_link_from_aka_ms() {
             return 1
         fi
 
+        sanitized_redirect_url=$(sanitize_redirect_url "$aka_ms_download_link")
+        if [[ -n "$sanitized_redirect_url" ]]; then
+            aka_ms_download_link="$sanitized_redirect_url"
+        fi
+
         say_verbose "The redirect location retrieved: '$aka_ms_download_link'."
         return 0
     else
@@ -1193,7 +1395,9 @@ get_download_link_from_aka_ms() {
 get_feeds_to_use()
 {
     feeds=(
+    "https://builds.dotnet.microsoft.com/dotnet"
     "https://dotnetcli.azureedge.net/dotnet"
+    "https://ci.dot.net/public"
     "https://dotnetbuilds.azureedge.net/public"
     )
 
@@ -1419,10 +1623,11 @@ install_dotnet() {
     eval $invocation
     local download_failed=false
     local download_completed=false
+    local remote_file_size=0
 
     mkdir -p "$install_root"
-    zip_path="$(mktemp "$temporary_file_template")"
+    zip_path="${zip_path:-$(mktemp "$temporary_file_template")}"
-    say_verbose "Zip path: $zip_path"
+    say_verbose "Archive path: $zip_path"
 
     for link_index in "${!download_links[@]}"
     do
@@ -1446,7 +1651,7 @@ install_dotnet() {
                 say "Failed to download $link_type link '$download_link': $download_error_msg"
                 ;;
             esac
-            rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed"
+            rm -f "$zip_path" 2>&1 && say_verbose "Temporary archive file $zip_path was removed"
         else
             download_completed=true
             break
@@ -1459,8 +1664,10 @@ install_dotnet() {
         return 1
     fi
 
-    say "Extracting zip from $download_link"
+    remote_file_size="$(get_remote_file_size "$download_link")"
-    extract_dotnet_package "$zip_path" "$install_root" || return 1
+
+    say "Extracting archive from $download_link"
+    extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1
 
     #  Check if the SDK version is installed; if not, fail the installation.
     # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed.
@@ -1610,10 +1817,22 @@ do
             override_non_versioned_files=false
             non_dynamic_parameters+=" $name"
             ;;
+        --keep-zip|-[Kk]eep[Zz]ip)
+            keep_zip=true
+            non_dynamic_parameters+=" $name"
+            ;;
+        --zip-path|-[Zz]ip[Pp]ath)
+            shift
+            zip_path="$1"
+            ;;
         -?|--?|-h|--help|-[Hh]elp)
-            script_name="$(basename "$0")"
+            script_name="dotnet-install.sh"
             echo ".NET Tools Installer"
-            echo "Usage: $script_name [-c|--channel <CHANNEL>] [-v|--version <VERSION>] [-p|--prefix <DESTINATION>]"
+            echo "Usage:"
+            echo "       # Install a .NET SDK of a given Quality from a given Channel"
+            echo "       $script_name [-c|--channel <CHANNEL>] [-q|--quality <QUALITY>]"
+            echo "       # Install a .NET SDK of a specific public version"
+            echo "       $script_name [-v|--version <VERSION>]"
             echo "       $script_name -h|-?|--help"
             echo ""
             echo "$script_name is a simple command line interface for obtaining dotnet cli."
@@ -1655,7 +1874,7 @@ do
             echo "      -InstallDir"
             echo "  --architecture <ARCHITECTURE>      Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`."
             echo "      --arch,-Architecture,-Arch"
-            echo "          Possible values: x64, arm, arm64 and s390x"
+            echo "          Possible values: x64, arm, arm64, s390x, ppc64le and loongarch64"
             echo "  --os <system>                    Specifies operating system to be used when selecting the installer."
             echo "          Overrides the OS determination approach used by the script. Supported values: osx, linux, linux-musl, freebsd, rhel.6."
             echo "          In case any other value is provided, the platform will be determined by the script based on machine configuration."
@@ -1680,6 +1899,8 @@ do
             echo "  --no-cdn,-NoCdn                    Disable downloading from the Azure CDN, and use the uncached feed directly."
             echo "  --jsonfile <JSONFILE>              Determines the SDK version from a user specified global.json file."
             echo "                                     Note: global.json must have a value for 'SDK:Version'"
+            echo "  --keep-zip,-KeepZip                If set, downloaded file is kept."
+            echo "  --zip-path, -ZipPath               If set, downloaded file is stored at the specified path."
             echo "  -?,--?,-h,--help,-Help             Shows this help message"
             echo ""
             echo "Install Location:"

src/installer.ts

@@ -101,9 +101,18 @@ export class DotnetVersionResolver {
       allowRetries: true,
       maxRetries: 3
     });
-    const response = await httpClient.getJson<any>(
+
+    let response;
+    try {
+      response = await httpClient.getJson<any>(
         DotnetVersionResolver.DotNetCoreIndexUrl
       );
+    } catch (error) {
+      response = await httpClient.getJson<any>(
+        DotnetVersionResolver.DotnetCoreIndexFallbackUrl
+      );
+    }
+
     const result = response.result || {};
     const releasesInfo: any[] = result['releases-index'];
 
@@ -122,6 +131,9 @@ export class DotnetVersionResolver {
   }
 
   static DotNetCoreIndexUrl =
+    'https://builds.dotnet.microsoft.com/dotnet/release-metadata/releases-index.json';
+
+  static DotnetCoreIndexFallbackUrl =
     'https://dotnetcli.azureedge.net/dotnet/release-metadata/releases-index.json';
 }