Add retry and timeout for stuck brew calls

src/scripts/darwin.sh

@@ -15,7 +15,7 @@ handle_dependency_extensions() {
     brew_opts=(-sf)
     patch_abstract_file >/dev/null 2>&1
     for dependency_extension in "${dependency_extensions[@]}"; do
-        brew install "${brew_opts[@]}" "$ext_tap/$dependency_extension@$version" >/dev/null 2>&1 && copy_brew_extensions "$dependency_extension"
+        safe_brew install "${brew_opts[@]}" "$ext_tap/$dependency_extension@$version" >/dev/null 2>&1 && copy_brew_extensions "$dependency_extension"
     done
   fi
 }
@@ -83,7 +83,7 @@ add_brew_extension() {
     formula="$(get_renamed_formula "$formula")"
     update_dependencies >/dev/null 2>&1
     handle_dependency_extensions "$formula" "$extension" >/dev/null 2>&1
-    (brew install "${brew_opts[@]}" "$ext_tap/$formula@$version" >/dev/null 2>&1 && copy_brew_extensions "$formula") || pecl_install "$extension" >/dev/null 2>&1
+    (safe_brew install "${brew_opts[@]}" "$ext_tap/$formula@$version" >/dev/null 2>&1 && copy_brew_extensions "$formula") || pecl_install "$extension" >/dev/null 2>&1
     add_extension_log "$extension" "Installed and enabled"
   fi
 }
@@ -181,14 +181,14 @@ add_php() {
   fi
   if [[ "$existing_version" != "false" && -z "$suffix" ]]; then
     if [ "$action" = "upgrade" ]; then
-      brew install --only-dependencies "$php_formula"
+      safe_brew install --only-dependencies "$php_formula"
-      brew upgrade -f --overwrite "$php_formula"
+      safe_brew upgrade -f --overwrite "$php_formula"
     else
       brew unlink "$php_keg"
     fi
   else
-    brew install --only-dependencies "$php_formula"
+    safe_brew install --only-dependencies "$php_formula"
-    brew install -f --overwrite "$php_formula" 2>/dev/null || brew upgrade -f --overwrite "$php_formula"
+    safe_brew install -f --overwrite "$php_formula" 2>/dev/null || safe_brew upgrade -f --overwrite "$php_formula"
   fi
   brew link --force --overwrite "$php_keg" || (sudo chown -R "$(id -un)":"$(id -gn)" "$brew_prefix" && brew link --force --overwrite "$php_keg")
 }

src/scripts/extensions/relay.sh

@@ -36,7 +36,7 @@ get_openssl_suffix() {
 change_library_paths() {
   if [ "$os" = "Darwin" ]; then
     otool -L "${ext_dir:?}"/relay.so | grep -q 'ssl.1' && openssl_version='1.1' || openssl_version='3'
-    [ -e "${brew_prefix:?}"/opt/openssl@"$openssl_version" ] || brew install openssl@"$openssl_version"
+    [ -e "${brew_prefix:?}"/opt/openssl@"$openssl_version" ] || safe_brew install openssl@"$openssl_version"
     dylibs="$(otool -L "${ext_dir:?}"/relay.so | grep -Eo '.*\.dylib' | cut -f1 -d ' ')"
     install_name_tool -change "$(echo "${dylibs}" | grep -E "libzstd.*dylib" | xargs)" "$brew_prefix"/opt/zstd/lib/libzstd.dylib "$ext_dir"/relay.so
     install_name_tool -change "$(echo "${dylibs}" | grep -E "liblz4.*dylib" | xargs)" "$brew_prefix"/opt/lz4/lib/liblz4.dylib "$ext_dir"/relay.so
@@ -54,7 +54,7 @@ add_relay_dependencies() {
   if [ "$os" = "Darwin" ]; then
     . "${0%/*}"/tools/brew.sh
     configure_brew
-    brew install lz4 hiredis zstd concurrencykit
+    safe_brew install lz4 hiredis zstd concurrencykit
   fi
 }
 

src/scripts/extensions/source.sh

@@ -60,7 +60,7 @@ add_linux_libs() {
 add_darwin_libs() {
   local lib=$1
   if ! check_lib "$lib"; then
-    brew install "$lib" >/dev/null 2>&1 || true
+    safe_brew install "$lib" >/dev/null 2>&1 || true
     if [[ "$lib" = *@* ]]; then
       brew link --overwrite --force "$lib" >/dev/null 2>&1 || true
     fi

src/scripts/tools/blackfire.sh

@@ -8,7 +8,7 @@ add_blackfire_linux() {
 add_blackfire_darwin() {
   sudo mkdir -p /usr/local/var/run
   add_brew_tap blackfireio/homebrew-blackfire
-  brew install blackfire
+  safe_brew install blackfire
 }
 
 blackfire_config() {

src/scripts/tools/brew.sh

@@ -44,6 +44,118 @@ add_brew_bins_to_path() {
   add_path "$brew_prefix"/sbin
 }
 
+# Function to get file modification time.
+get_file_mtime() {
+  local file=$1
+  if [ "$(uname -s)" = "Darwin" ]; then
+    stat -f "%m" "$file" 2>/dev/null || echo 0
+  else
+    stat -c "%Y" "$file" 2>/dev/null || echo 0
+  fi
+}
+
+# Function to terminate a process and its direct children.
+terminate_process_tree() {
+  local pid=$1
+  local children child
+  children=$(pgrep -P "$pid" 2>/dev/null || true)
+  kill -TERM "$pid" >/dev/null 2>&1 || true
+  for child in $children; do
+    terminate_process_tree "$child"
+  done
+  sleep 2
+  kill -KILL "$pid" >/dev/null 2>&1 || true
+  for child in $children; do
+    terminate_process_tree "$child"
+  done
+}
+
+# Function to run a command with an inactivity watchdog.
+run_with_inactivity_watchdog() {
+  local timeout_secs="${SETUP_PHP_BREW_INACTIVITY_TIMEOUT:-180}"
+  local poll_secs="${SETUP_PHP_BREW_WATCHDOG_POLL:-5}"
+  local tmp_dir fifo log_file timeout_file command_pid reader_pid monitor_pid exit_code
+  tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/setup-php-brew.XXXXXX")" || return 1
+  fifo="$tmp_dir/output.fifo"
+  log_file="$tmp_dir/output.log"
+  timeout_file="$tmp_dir/timed_out"
+  mkfifo "$fifo" || {
+    rm -rf "$tmp_dir"
+    return 1
+  }
+  : >"$log_file"
+
+  ("$@" >"$fifo" 2>&1) &
+  command_pid=$!
+
+  (
+    while IFS= read -r line || [ -n "$line" ]; do
+      printf '%s\n' "$line"
+      printf '%s\n' "$line" >>"$log_file"
+    done <"$fifo"
+  ) &
+  reader_pid=$!
+
+  (
+    local last_activity current_activity now
+    last_activity=$(get_file_mtime "$log_file")
+    while kill -0 "$command_pid" >/dev/null 2>&1; do
+      sleep "$poll_secs"
+      current_activity=$(get_file_mtime "$log_file")
+      [ "$current_activity" -gt "$last_activity" ] && last_activity="$current_activity"
+      now=$(date +%s)
+      if [ $((now - last_activity)) -ge "$timeout_secs" ]; then
+        printf "\nsetup-php: brew produced no output for %ss; terminating and retrying...\n" "$timeout_secs" >&2
+        : >"$timeout_file"
+        terminate_process_tree "$command_pid"
+        break
+      fi
+    done
+  ) &
+  monitor_pid=$!
+
+  wait "$command_pid"
+  exit_code=$?
+  wait "$reader_pid" 2>/dev/null || true
+  kill "$monitor_pid" >/dev/null 2>&1 || true
+  wait "$monitor_pid" 2>/dev/null || true
+
+  if [ -e "$timeout_file" ]; then
+    rm -rf "$tmp_dir"
+    return 124
+  fi
+
+  rm -rf "$tmp_dir"
+  return "$exit_code"
+}
+
+# Function to run brew with retries and an inactivity watchdog.
+safe_brew() {
+  local max_attempts="${SETUP_PHP_BREW_RETRY_ATTEMPTS:-3}"
+  local attempt=1
+  local exit_code=0
+
+  if [ "${SETUP_PHP_BREW_WATCHDOG:-true}" = "false" ]; then
+    brew "$@"
+    return $?
+  fi
+
+  while [ "$attempt" -le "$max_attempts" ]; do
+    run_with_inactivity_watchdog brew "$@" && return 0
+    exit_code=$?
+
+    if [ "$attempt" -ge "$max_attempts" ]; then
+      return "$exit_code"
+    fi
+
+    printf "setup-php: retrying brew command (attempt %s/%s, exit %s)\n" "$((attempt + 1))" "$max_attempts" "$exit_code" >&2
+    sleep "$((attempt * 5))"
+    attempt=$((attempt + 1))
+  done
+
+  return "$exit_code"
+}
+
 # Function to add brew.
 add_brew() {
   brew_prefix="$(get_brew_prefix)"
@@ -74,6 +186,7 @@ configure_brew() {
   export HOMEBREW_NO_ENV_HINTS=1
   export HOMEBREW_NO_INSTALL_CLEANUP=1
   export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+  export HOMEBREW_DOWNLOAD_CONCURRENCY="${HOMEBREW_DOWNLOAD_CONCURRENCY:-6}"
   export brew_opts
   export brew_path
   export brew_path_dir

src/scripts/tools/grpc_php_plugin.sh

@@ -4,7 +4,7 @@ add_bazel() {
       add_list bazel/apt https://storage.googleapis.com/bazel-apt https://bazel.build/bazel-release.pub.gpg stable jdk1.8
       install_packages bazel
     else
-      brew install bazel
+      safe_brew install bazel
     fi
   fi
 }
@@ -25,7 +25,7 @@ add_grpc_php_plugin_brew() {
   . "${0%/*}"/tools/brew.sh
   configure_brew
   [ -e /usr/local/bin/protoc ] && sudo mv /usr/local/bin/protoc /tmp/protoc && sudo mv /usr/local/include/google /tmp
-  brew install grpc
+  safe_brew install grpc
   brew link --force --overwrite grpc >/dev/null 2>&1
   [ -e /tmp/protoc ] && sudo mv /tmp/protoc /usr/local/bin/protoc && sudo mv /tmp/google /usr/local/include/
   grpc_tag="v$(brew info grpc | grep "grpc:" | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+")"