From c6e99860cd6f5e212ffcb74097a6722bba706825 Mon Sep 17 00:00:00 2001 From: Shivam Mathur Date: Tue, 10 Aug 2021 01:37:46 +0530 Subject: [PATCH] Improve adding PPAs on Linux --- src/configs/os_releases.csv | 17 +++ src/scripts/ext/phalcon.sh | 5 +- src/scripts/linux.sh | 48 ++------- src/scripts/tools/blackfire.sh | 7 +- src/scripts/tools/grpc_php_plugin.sh | 7 +- src/scripts/tools/ppa.sh | 151 +++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 54 deletions(-) create mode 100644 src/configs/os_releases.csv create mode 100644 src/scripts/tools/ppa.sh diff --git a/src/configs/os_releases.csv b/src/configs/os_releases.csv new file mode 100644 index 00000000..62337b3b --- /dev/null +++ b/src/configs/os_releases.csv @@ -0,0 +1,17 @@ +8,jessie +9,stretch +10,buster +11,bullseye +12,bookworm +16.04 LTS,xenial +16.10,yakkety +17.04,zesty +17.10,artful +18.04 LTS,bionic +18.10,cosmic +19.04,disco +19.10,eoan +20.04 LTS,focal +20.10,groovy +21.04,hirsute +21.10,impish diff --git a/src/scripts/ext/phalcon.sh b/src/scripts/ext/phalcon.sh index 48de1c16..72dfc9be 100644 --- a/src/scripts/ext/phalcon.sh +++ b/src/scripts/ext/phalcon.sh @@ -1,11 +1,10 @@ # Helper function to add phalcon. add_phalcon_helper() { status='Installed and enabled' - update_lists if [ "$extension" = "phalcon4" ]; then - ${apt_install:?} "php${version:?}-psr" "php${version:?}-$extension" + install_packages "php${version:?}-psr" "php${version:?}-$extension" else - ${apt_install:?} "php${version:?}-$extension" + install_packages "php${version:?}-$extension" fi } diff --git a/src/scripts/linux.sh b/src/scripts/linux.sh index 63075817..d1c1ce82 100644 --- a/src/scripts/linux.sh +++ b/src/scripts/linux.sh @@ -1,56 +1,21 @@ # Function to setup environment for self-hosted runners. self_hosted_helper() { if ! command -v sudo >/dev/null; then + apt-get update apt-get install -y sudo || add_log "${cross:?}" "sudo" "Could not install sudo" fi if ! command -v apt-fast >/dev/null; then sudo ln -sf /usr/bin/apt-get /usr/bin/apt-fast trap "sudo rm -f /usr/bin/apt-fast 2>/dev/null" exit fi - install_packages apt-transport-https ca-certificates curl make software-properties-common unzip autoconf automake gcc g++ gnupg - add_ppa ondrej/php -} - -# Function to backup and cleanup package lists. -cleanup_lists() { - ppa_prefix=${1-ondrej} - if [ ! -e /etc/apt/sources.list.d.save ]; then - sudo mv /etc/apt/sources.list.d /etc/apt/sources.list.d.save - sudo mkdir /etc/apt/sources.list.d - sudo mv /etc/apt/sources.list.d.save/*"${ppa_prefix}"*.list /etc/apt/sources.list.d/ - trap "sudo mv /etc/apt/sources.list.d.save/*.list /etc/apt/sources.list.d/ 2>/dev/null" exit - fi -} - -# Function to add ppa:ondrej/php. -add_ppa() { - ppa=${1:-ondrej/php} - if [ "$ID" = "debian" ] && [ "$ppa" = "ondrej/php" ]; then - get -q -n /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg - echo "deb https://packages.sury.org/php/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/ondrej.list - elif ! apt-cache policy | grep -q "$ppa"; then - cleanup_lists "$(dirname "$ppa")" - LC_ALL=C.UTF-8 sudo apt-add-repository ppa:"$ppa" -y - fi - if [ "$ID" = "debian" ]; then - sudo "$debconf_fix" apt-get update - fi -} - -# Function to update the package lists. -update_lists() { - if [ ! -e /tmp/setup_php ]; then - [ "${runner:?}" != "self-hosted" ] && add_ppa >/dev/null 2>&1 - cleanup_lists - sudo "$debconf_fix" apt-get update >/dev/null 2>&1 - echo '' | sudo tee /tmp/setup_php >/dev/null 2>&1 - fi + install_packages apt-transport-https ca-certificates curl file make jq unzip autoconf automake gcc g++ gnupg } # Function to install a package install_packages() { packages=("$@") - $apt_install "${packages[@]}" >/dev/null 2>&1 || update_lists && $apt_install "${packages[@]}" >/dev/null 2>&1 + [[ "${packages[*]}" =~ php ]] && add_ppa ondrej/php + $apt_install "${packages[@]}" >/dev/null 2>&1 || (update_lists && $apt_install "${packages[@]}" >/dev/null 2>&1) } # Function to disable an extension. @@ -146,7 +111,7 @@ add_devtools() { # Function to setup the nightly build from shivammathur/php-builder setup_nightly() { - run_script "php-builder" "$runner" "$version" + run_script "php-builder" "${runner:?}" "$version" } # Function to setup PHP 5.3, PHP 5.4 and PHP 5.5. @@ -183,7 +148,7 @@ add_packaged_php() { if [ "$runner" = "self-hosted" ] || [ "${use_package_cache:-true}" = "false" ]; then update_lists IFS=' ' read -r -a packages <<<"$(echo "cli curl mbstring xml intl" | sed "s/[^ ]*/php$version-&/g")" - $apt_install "${packages[@]}" + install_packages "${packages[@]}" else run_script "php-ubuntu" "$version" fi @@ -283,6 +248,7 @@ scripts="${dist}"/../src/scripts . /etc/os-release # shellcheck source=. . "${scripts:?}"/ext/source.sh +. "${scripts:?}"/tools/ppa.sh . "${scripts:?}"/tools/add_tools.sh . "${scripts:?}"/common.sh read_env diff --git a/src/scripts/tools/blackfire.sh b/src/scripts/tools/blackfire.sh index 672be674..b6886cfb 100644 --- a/src/scripts/tools/blackfire.sh +++ b/src/scripts/tools/blackfire.sh @@ -1,10 +1,7 @@ add_blackfire_linux() { sudo mkdir -p /var/run/blackfire - get -s -n "" https://packages.blackfire.io/gpg.key | sudo apt-key add - - echo "deb http://packages.blackfire.io/debian any main" | sudo tee /etc/apt/sources.list.d/blackfire.list - sudo "${debconf_fix:?}" apt-get update - ${apt_install:?} blackfire - sudo chmod -R 777 /var/run/blackfire /etc/blackfire/agent + add_list debian/blackfire http://packages.blackfire.io/debian https://packages.blackfire.io/gpg.key any main + install_packages blackfire } add_blackfire_darwin() { diff --git a/src/scripts/tools/grpc_php_plugin.sh b/src/scripts/tools/grpc_php_plugin.sh index 54c15eb8..2452bf9a 100644 --- a/src/scripts/tools/grpc_php_plugin.sh +++ b/src/scripts/tools/grpc_php_plugin.sh @@ -1,11 +1,8 @@ add_bazel() { if ! command -v bazel; then if [ "$(uname -s)" = "Linux" ]; then - ${apt_install:?} curl gnupg - get -s -n "" https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - - echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list - sudo "${debconf_fix:?}" apt-get update -y - ${apt_install:?} bazel + add_list bazel/apt https://storage.googleapis.com/bazel-apt https://bazel.build/bazel-release.pub.gpg stable jdk1.8 + install_packages bazel else brew install bazel fi diff --git a/src/scripts/tools/ppa.sh b/src/scripts/tools/ppa.sh new file mode 100644 index 00000000..8a7f388e --- /dev/null +++ b/src/scripts/tools/ppa.sh @@ -0,0 +1,151 @@ +# Function to try to set ubuntu or debian version. +set_base_version_id() { + [[ "$ID" =~ ubuntu|debian ]] && return; + if ! [ -d "$dist_info_dir" ]; then + sudo mkdir -p "$dist_info_dir" + get -q -n "$dist_info_dir"/os_releases.csv https://raw.githubusercontent.com/shivammathur/setup-php/develop/src/configs/os_releases.csv + fi + for base in ubuntu debian; do + [[ "$ID_LIKE" =~ $base ]] && ID="$base" && VERSION_ID="$(grep -hr -m 1 "$VERSION_CODENAME" /usr/share/distro-info | cut -d ',' -f 1 | cut -d ' ' -f 1)" && break + done +} + +# Function to try to set ubuntu or debian codename. +set_base_version_codename() { + [[ "$ID" =~ ubuntu|debian ]] && return; + if [[ "$ID_LIKE" =~ ubuntu ]]; then + [[ -n "$UBUNTU_CODENAME" ]] && VERSION_CODENAME="$UBUNTU_CODENAME" && return; + [ -e "$upstream_lsb" ] && VERSION_CODENAME=$(grep 'CODENAME' "$upstream_lsb" | cut -d '=' -f 2) && return; + VERSION_CODENAME=$(grep -E -m1 'deb .*ubuntu.com' "$list_file" | cut -d ' ' -f 3) && VERSION_CODENAME=${VERSION_CODENAME%-*} + elif [[ "$ID_LIKE" =~ debian ]] || command -v dpkg >/dev/null; then + ID_LIKE=debian + [[ -n "$DEBIAN_CODENAME" ]] && VERSION_CODENAME="$DEBIAN_CODENAME" && return; + update_lists && VERSION_CODENAME=$(apt-cache show tzdata | grep -m 1 Provides | cut -d '-' -f 2) + fi +} + +# Function to set base os details +set_base_version() { + if [ -e /tmp/os-release ]; then + . /tmp/os-release + else + set_base_version_codename + set_base_version_id + printf "ID=%s\nVERSION_ID=%s\nVERSION_CODENAME=%s\n" "$ID" "$VERSION_ID" "$VERSION_CODENAME" | tee /tmp/os-release >/dev/null 2>&1 + fi +} + +# Helper function to update package lists. +update_lists_helper() { + list=$1 + command -v sudo >/dev/null && SUDO=sudo + if [[ -n "$list" ]]; then + ${SUDO} apt-get update -o Dir::Etc::sourcelist="$list" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" + else + ${SUDO} apt-get update + fi +} + +# Function to update the package lists. +update_lists() { + local ppa=${1:-} + local ppa_search=${2:-} + if [ ! -e /tmp/setup_php ] || [[ -n $ppa && -n $ppa_search ]]; then + if [[ -n "$ppa" && -n "$ppa_search" ]]; then + list="$list_dir"/"$(basename "$(grep -lr "$ppa_search" "$list_dir")")" + elif grep -Eq '^deb ' "$list_file"; then + list="$list_file" + fi + update_lists_helper "$list" + echo '' | tee /tmp/setup_php >/dev/null 2>&1 + fi +} + +# Function to get fingerprint from an Ubuntu PPA. +ubuntu_fingerprint() { + ppa=$1 + get -s -n "" "$lp_api"/~"${ppa%/*}"/+archive/"${ppa##*/}" | jq -r '.signing_key_fingerprint' +} + +# Function to get fingerprint from a Debian PPA. +debian_fingerprint() { + ppa=$1 + ppa_url=$2 + package_dist=$3 + release_pub=/tmp/"${ppa/\//-}".gpg + get -q -n "$release_pub" "$ppa_url"/dists/"$package_dist"/Release.gpg + gpg --list-packets "$release_pub" | grep -Eo 'fpr\sv4\s.*[a-zA-Z0-9]+' | head -n 1 | cut -d ' ' -f 3 +} + +# Function to add a GPG key. +add_key() { + ppa=${1:-ondrej/php} + ppa_url=$2 + package_dist=$3 + key_source=$4 + key_file=$5 + key_urls=("$key_source") + if [[ "$key_source" =~ launchpad.net|debian.org|setup-php.com ]]; then + fingerprint="$("${ID}"_fingerprint "$ppa" "$ppa_url" "$package_dist")" + sks_params="op=get&options=mr&exact=on&search=0x$fingerprint" + key_urls=("${sks[@]/%/\/pks\/lookup\?"$sks_params"}") + fi + [ ! -e "$key_source" ] && get -q -n "$key_file" "${key_urls[@]}" + if [[ "$(file "$key_file")" =~ .*('Public-Key (old)'|'Secret-Key') ]]; then + sudo gpg --batch --yes --dearmor "$key_file" >/dev/null 2>&1 && sudo mv "$key_file".gpg "$key_file" + fi +} + +# Function to add a sources list. +add_list() { + ppa=${1-ondrej/php} + ppa_url=${2:-"$lp_ppa/$ppa/ubuntu"} + key_source=${3:-"$ppa_url"} + package_dist=${4:-"$VERSION_CODENAME"} + branches=${5:-main} + ppa_search="deb .*$ppa_url $package_dist .*$branches" + grep -Eqr "$ppa_search" "$list_dir" && echo "Repository $ppa already exists" && return; + arch=$(dpkg --print-architecture) + [ -e "$key_source" ] && key_file=$key_source || key_file="$key_dir"/"${ppa/\//-}"-keyring.gpg + add_key "$ppa" "$ppa_url" "$package_dist" "$key_source" "$key_file" + echo "deb [arch=$arch signed-by=$key_file] $ppa_url $package_dist $branches" | sudo tee -a "$list_dir"/"${ppa/\//-}".list >/dev/null 2>&1 + update_lists "$ppa" "$ppa_search" + . /etc/os-release +} + +# Function to remove a PPA. +remove_list() { + ppa=${1-ondrej/php} + ppa_url=${2:-"$lp_ppa/$ppa/ubuntu"} + grep -lr "$ppa_url" "$list_dir" | xargs -n1 sudo rm -f + sudo rm -f "$key_dir"/"${ppa/\//-}"-keyring || true +} + +# Function to add a PPA. +add_ppa() { + set_base_version + ppa=${1:-ondrej/php} + if [[ "$ID" = "ubuntu" || "$ID_LIKE" =~ ubuntu ]] && [[ "$ppa" =~ "ondrej/" ]]; then + add_list "$ppa" + elif [[ "$ID" = "debian" || "$ID_LIKE" =~ debian ]] && [[ "$ppa" =~ "ondrej/" ]]; then + add_list "$ppa" "$sury"/"${ppa##*/}"/ "$sury"/"${ppa##*/}"/apt.gpg + else + add_list "$ppa" + fi + . /etc/os-release +} + +# Variables +list_file='/etc/apt/sources.list' +list_dir="$list_file.d" +upstream_lsb='/etc/upstream-release/lsb-release' +lp_api='https://api.launchpad.net/1.0' +lp_ppa='http://ppa.launchpad.net' +key_dir='/usr/share/keyrings' +dist_info_dir='/usr/share/distro-info' +sury='https://packages.sury.org' +sks=( + 'https://keyserver.ubuntu.com' + 'https://pgp.mit.edu' + 'https://keys.openpgp.org' +)