Improve adding PPAs on Linux

src/configs/os_releases.csv

@@ -0,0 +1,17 @@
+8,jessie
+9,stretch
+10,buster
+11,bullseye
+12,bookworm
+16.04 LTS,xenial
+16.10,yakkety
+17.04,zesty
+17.10,artful
+18.04 LTS,bionic
+18.10,cosmic
+19.04,disco
+19.10,eoan
+20.04 LTS,focal
+20.10,groovy
+21.04,hirsute
+21.10,impish

src/scripts/ext/phalcon.sh

@@ -1,11 +1,10 @@
 # Helper function to add phalcon.
 add_phalcon_helper() {
   status='Installed and enabled'
-  update_lists
   if [ "$extension" = "phalcon4" ]; then
-    ${apt_install:?} "php${version:?}-psr" "php${version:?}-$extension"
+    install_packages "php${version:?}-psr" "php${version:?}-$extension"
   else
-    ${apt_install:?} "php${version:?}-$extension"
+    install_packages "php${version:?}-$extension"
   fi
 }
 

src/scripts/linux.sh

@@ -1,56 +1,21 @@
 # Function to setup environment for self-hosted runners.
 self_hosted_helper() {
   if ! command -v sudo >/dev/null; then
+    apt-get update
     apt-get install -y sudo || add_log "${cross:?}" "sudo" "Could not install sudo"
   fi
   if ! command -v apt-fast >/dev/null; then
     sudo ln -sf /usr/bin/apt-get /usr/bin/apt-fast
     trap "sudo rm -f /usr/bin/apt-fast 2>/dev/null" exit
   fi
-  install_packages apt-transport-https ca-certificates curl make software-properties-common unzip autoconf automake gcc g++ gnupg
-  add_ppa ondrej/php
-}
-
-# Function to backup and cleanup package lists.
-cleanup_lists() {
-  ppa_prefix=${1-ondrej}
-  if [ ! -e /etc/apt/sources.list.d.save ]; then
-    sudo mv /etc/apt/sources.list.d /etc/apt/sources.list.d.save
-    sudo mkdir /etc/apt/sources.list.d
-    sudo mv /etc/apt/sources.list.d.save/*"${ppa_prefix}"*.list /etc/apt/sources.list.d/
-    trap "sudo mv /etc/apt/sources.list.d.save/*.list /etc/apt/sources.list.d/ 2>/dev/null" exit
-  fi
-}
-
-# Function to add ppa:ondrej/php.
-add_ppa() {
-  ppa=${1:-ondrej/php}
-  if [ "$ID" = "debian" ] && [ "$ppa" = "ondrej/php" ]; then
-    get -q -n /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
-    echo "deb https://packages.sury.org/php/ $VERSION_CODENAME main" > /etc/apt/sources.list.d/ondrej.list
-  elif ! apt-cache policy | grep -q "$ppa"; then
-    cleanup_lists "$(dirname "$ppa")"
-    LC_ALL=C.UTF-8 sudo apt-add-repository ppa:"$ppa" -y
-  fi
-  if [ "$ID" = "debian" ]; then
-    sudo "$debconf_fix" apt-get update
-  fi
-}
-
-# Function to update the package lists.
-update_lists() {
-  if [ ! -e /tmp/setup_php ]; then
-    [ "${runner:?}" != "self-hosted" ] && add_ppa >/dev/null 2>&1
-    cleanup_lists
-    sudo "$debconf_fix" apt-get update >/dev/null 2>&1
-    echo '' | sudo tee /tmp/setup_php >/dev/null 2>&1
-  fi
+  install_packages apt-transport-https ca-certificates curl file make jq unzip autoconf automake gcc g++ gnupg
 }
 
 # Function to install a package
 install_packages() {
   packages=("$@")
-  $apt_install "${packages[@]}" >/dev/null 2>&1 || update_lists && $apt_install "${packages[@]}" >/dev/null 2>&1
+  [[ "${packages[*]}" =~ php ]] && add_ppa ondrej/php
+  $apt_install "${packages[@]}" >/dev/null 2>&1 || (update_lists && $apt_install "${packages[@]}" >/dev/null 2>&1)
 }
 
 # Function to disable an extension.
@@ -146,7 +111,7 @@ add_devtools() {
 
 # Function to setup the nightly build from shivammathur/php-builder
 setup_nightly() {
-  run_script "php-builder" "$runner" "$version"
+  run_script "php-builder" "${runner:?}" "$version"
 }
 
 # Function to setup PHP 5.3, PHP 5.4 and PHP 5.5.
@@ -183,7 +148,7 @@ add_packaged_php() {
   if [ "$runner" = "self-hosted" ] || [ "${use_package_cache:-true}" = "false" ]; then
     update_lists
     IFS=' ' read -r -a packages <<<"$(echo "cli curl mbstring xml intl" | sed "s/[^ ]*/php$version-&/g")"
-    $apt_install "${packages[@]}"
+    install_packages "${packages[@]}"
   else
     run_script "php-ubuntu" "$version"
   fi
@@ -283,6 +248,7 @@ scripts="${dist}"/../src/scripts
 . /etc/os-release
 # shellcheck source=.
 . "${scripts:?}"/ext/source.sh
+. "${scripts:?}"/tools/ppa.sh
 . "${scripts:?}"/tools/add_tools.sh
 . "${scripts:?}"/common.sh
 read_env

src/scripts/tools/blackfire.sh

@@ -1,10 +1,7 @@
 add_blackfire_linux() {
   sudo mkdir -p /var/run/blackfire
-  get -s -n "" https://packages.blackfire.io/gpg.key | sudo apt-key add -
-  echo "deb http://packages.blackfire.io/debian any main" | sudo tee /etc/apt/sources.list.d/blackfire.list
-  sudo "${debconf_fix:?}" apt-get update
-  ${apt_install:?} blackfire
-  sudo chmod -R 777 /var/run/blackfire /etc/blackfire/agent
+  add_list debian/blackfire http://packages.blackfire.io/debian https://packages.blackfire.io/gpg.key any main
+  install_packages blackfire
 }
 
 add_blackfire_darwin() {

src/scripts/tools/grpc_php_plugin.sh

@@ -1,11 +1,8 @@
 add_bazel() {
   if ! command -v bazel; then
     if [ "$(uname -s)" = "Linux" ]; then
-      ${apt_install:?} curl gnupg
-      get -s -n "" https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
-      echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
-      sudo "${debconf_fix:?}" apt-get update -y
-      ${apt_install:?} bazel
+      add_list bazel/apt https://storage.googleapis.com/bazel-apt https://bazel.build/bazel-release.pub.gpg stable jdk1.8
+      install_packages bazel
     else
       brew install bazel
     fi

src/scripts/tools/ppa.sh

@@ -0,0 +1,151 @@
+# Function to try to set ubuntu or debian version.
+set_base_version_id() {
+  [[ "$ID" =~ ubuntu|debian ]] && return;
+  if ! [ -d "$dist_info_dir" ]; then
+    sudo mkdir -p "$dist_info_dir"
+    get -q -n "$dist_info_dir"/os_releases.csv https://raw.githubusercontent.com/shivammathur/setup-php/develop/src/configs/os_releases.csv
+  fi
+  for base in ubuntu debian; do
+    [[ "$ID_LIKE" =~ $base ]] && ID="$base" && VERSION_ID="$(grep -hr -m 1 "$VERSION_CODENAME" /usr/share/distro-info | cut -d ',' -f 1 | cut -d ' ' -f 1)" && break
+  done
+}
+
+# Function to try to set ubuntu or debian codename.
+set_base_version_codename() {
+  [[ "$ID" =~ ubuntu|debian ]] && return;
+  if [[ "$ID_LIKE" =~ ubuntu ]]; then
+    [[ -n "$UBUNTU_CODENAME" ]] && VERSION_CODENAME="$UBUNTU_CODENAME" && return;
+    [ -e "$upstream_lsb" ] && VERSION_CODENAME=$(grep 'CODENAME' "$upstream_lsb" | cut -d '=' -f 2) && return;
+    VERSION_CODENAME=$(grep -E -m1 'deb .*ubuntu.com' "$list_file" | cut -d ' ' -f 3) && VERSION_CODENAME=${VERSION_CODENAME%-*}
+  elif [[ "$ID_LIKE" =~ debian ]] || command -v dpkg >/dev/null; then
+    ID_LIKE=debian
+    [[ -n "$DEBIAN_CODENAME" ]] && VERSION_CODENAME="$DEBIAN_CODENAME" && return;
+    update_lists && VERSION_CODENAME=$(apt-cache show tzdata | grep -m 1 Provides | cut -d '-' -f 2)
+  fi
+}
+
+# Function to set base os details
+set_base_version() {
+  if [ -e /tmp/os-release ]; then
+    . /tmp/os-release
+  else
+    set_base_version_codename
+    set_base_version_id
+    printf "ID=%s\nVERSION_ID=%s\nVERSION_CODENAME=%s\n" "$ID" "$VERSION_ID" "$VERSION_CODENAME" | tee /tmp/os-release >/dev/null 2>&1
+  fi
+}
+
+# Helper function to update package lists.
+update_lists_helper() {
+  list=$1
+  command -v sudo >/dev/null && SUDO=sudo
+  if [[ -n "$list" ]]; then
+    ${SUDO} apt-get update -o Dir::Etc::sourcelist="$list" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0"
+  else
+    ${SUDO} apt-get update
+  fi
+}
+
+# Function to update the package lists.
+update_lists() {
+  local ppa=${1:-}
+  local ppa_search=${2:-}
+  if [ ! -e /tmp/setup_php ] || [[ -n $ppa && -n $ppa_search ]]; then
+    if [[ -n "$ppa" && -n "$ppa_search" ]]; then
+      list="$list_dir"/"$(basename "$(grep -lr "$ppa_search" "$list_dir")")"
+    elif grep -Eq '^deb ' "$list_file"; then
+      list="$list_file"
+    fi
+    update_lists_helper "$list"
+    echo '' | tee /tmp/setup_php >/dev/null 2>&1
+  fi
+}
+
+# Function to get fingerprint from an Ubuntu PPA.
+ubuntu_fingerprint() {
+  ppa=$1
+  get -s -n "" "$lp_api"/~"${ppa%/*}"/+archive/"${ppa##*/}" | jq -r '.signing_key_fingerprint'
+}
+
+# Function to get fingerprint from a Debian PPA.
+debian_fingerprint() {
+  ppa=$1
+  ppa_url=$2
+  package_dist=$3
+  release_pub=/tmp/"${ppa/\//-}".gpg
+  get -q -n "$release_pub" "$ppa_url"/dists/"$package_dist"/Release.gpg
+  gpg --list-packets "$release_pub" | grep -Eo 'fpr\sv4\s.*[a-zA-Z0-9]+' | head -n 1 | cut -d ' ' -f 3
+}
+
+# Function to add a GPG key.
+add_key() {
+  ppa=${1:-ondrej/php}
+  ppa_url=$2
+  package_dist=$3
+  key_source=$4
+  key_file=$5
+  key_urls=("$key_source")
+  if [[ "$key_source" =~ launchpad.net|debian.org|setup-php.com ]]; then
+    fingerprint="$("${ID}"_fingerprint "$ppa" "$ppa_url" "$package_dist")"
+    sks_params="op=get&options=mr&exact=on&search=0x$fingerprint"
+    key_urls=("${sks[@]/%/\/pks\/lookup\?"$sks_params"}")
+  fi
+  [ ! -e "$key_source" ] && get -q -n "$key_file" "${key_urls[@]}"
+  if [[ "$(file "$key_file")" =~ .*('Public-Key (old)'|'Secret-Key') ]]; then
+    sudo gpg --batch --yes --dearmor "$key_file" >/dev/null 2>&1 && sudo mv "$key_file".gpg "$key_file"
+  fi
+}
+
+# Function to add a sources list.
+add_list() {
+  ppa=${1-ondrej/php}
+  ppa_url=${2:-"$lp_ppa/$ppa/ubuntu"}
+  key_source=${3:-"$ppa_url"}
+  package_dist=${4:-"$VERSION_CODENAME"}
+  branches=${5:-main}
+  ppa_search="deb .*$ppa_url $package_dist .*$branches"
+  grep -Eqr "$ppa_search" "$list_dir" && echo "Repository $ppa already exists" && return;
+  arch=$(dpkg --print-architecture)
+  [ -e "$key_source" ] && key_file=$key_source || key_file="$key_dir"/"${ppa/\//-}"-keyring.gpg
+  add_key "$ppa" "$ppa_url" "$package_dist" "$key_source" "$key_file"
+  echo "deb [arch=$arch signed-by=$key_file] $ppa_url $package_dist $branches" | sudo tee -a "$list_dir"/"${ppa/\//-}".list >/dev/null 2>&1
+  update_lists "$ppa" "$ppa_search"
+  . /etc/os-release
+}
+
+# Function to remove a PPA.
+remove_list() {
+  ppa=${1-ondrej/php}
+  ppa_url=${2:-"$lp_ppa/$ppa/ubuntu"}
+  grep -lr "$ppa_url" "$list_dir" | xargs -n1 sudo rm -f
+  sudo rm -f "$key_dir"/"${ppa/\//-}"-keyring || true
+}
+
+# Function to add a PPA.
+add_ppa() {
+  set_base_version
+  ppa=${1:-ondrej/php}
+  if [[ "$ID" = "ubuntu" || "$ID_LIKE" =~ ubuntu ]] && [[ "$ppa" =~ "ondrej/" ]]; then
+    add_list "$ppa"
+  elif [[ "$ID" = "debian" || "$ID_LIKE" =~ debian ]] && [[ "$ppa" =~ "ondrej/" ]]; then
+    add_list "$ppa" "$sury"/"${ppa##*/}"/ "$sury"/"${ppa##*/}"/apt.gpg
+  else
+    add_list "$ppa"
+  fi
+  . /etc/os-release
+}
+
+# Variables
+list_file='/etc/apt/sources.list'
+list_dir="$list_file.d"
+upstream_lsb='/etc/upstream-release/lsb-release'
+lp_api='https://api.launchpad.net/1.0'
+lp_ppa='http://ppa.launchpad.net'
+key_dir='/usr/share/keyrings'
+dist_info_dir='/usr/share/distro-info'
+sury='https://packages.sury.org'
+sks=(
+  'https://keyserver.ubuntu.com'
+  'https://pgp.mit.edu'
+  'https://keys.openpgp.org'
+)