Add retry and timeout for stuck brew calls

src/scripts/tools/brew.sh

@@ -44,6 +44,118 @@ add_brew_bins_to_path() {
   add_path "$brew_prefix"/sbin
 }
 
+# Function to get file modification time.
+get_file_mtime() {
+  local file=$1
+  if [ "$(uname -s)" = "Darwin" ]; then
+    stat -f "%m" "$file" 2>/dev/null || echo 0
+  else
+    stat -c "%Y" "$file" 2>/dev/null || echo 0
+  fi
+}
+
+# Function to terminate a process and its direct children.
+terminate_process_tree() {
+  local pid=$1
+  local children child
+  children=$(pgrep -P "$pid" 2>/dev/null || true)
+  kill -TERM "$pid" >/dev/null 2>&1 || true
+  for child in $children; do
+    terminate_process_tree "$child"
+  done
+  sleep 2
+  kill -KILL "$pid" >/dev/null 2>&1 || true
+  for child in $children; do
+    terminate_process_tree "$child"
+  done
+}
+
+# Function to run a command with an inactivity watchdog.
+run_with_inactivity_watchdog() {
+  local timeout_secs="${SETUP_PHP_BREW_INACTIVITY_TIMEOUT:-180}"
+  local poll_secs="${SETUP_PHP_BREW_WATCHDOG_POLL:-5}"
+  local tmp_dir fifo log_file timeout_file command_pid reader_pid monitor_pid exit_code
+  tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/setup-php-brew.XXXXXX")" || return 1
+  fifo="$tmp_dir/output.fifo"
+  log_file="$tmp_dir/output.log"
+  timeout_file="$tmp_dir/timed_out"
+  mkfifo "$fifo" || {
+    rm -rf "$tmp_dir"
+    return 1
+  }
+  : >"$log_file"
+
+  ("$@" >"$fifo" 2>&1) &
+  command_pid=$!
+
+  (
+    while IFS= read -r line || [ -n "$line" ]; do
+      printf '%s\n' "$line"
+      printf '%s\n' "$line" >>"$log_file"
+    done <"$fifo"
+  ) &
+  reader_pid=$!
+
+  (
+    local last_activity current_activity now
+    last_activity=$(get_file_mtime "$log_file")
+    while kill -0 "$command_pid" >/dev/null 2>&1; do
+      sleep "$poll_secs"
+      current_activity=$(get_file_mtime "$log_file")
+      [ "$current_activity" -gt "$last_activity" ] && last_activity="$current_activity"
+      now=$(date +%s)
+      if [ $((now - last_activity)) -ge "$timeout_secs" ]; then
+        printf "\nsetup-php: brew produced no output for %ss; terminating and retrying...\n" "$timeout_secs" >&2
+        : >"$timeout_file"
+        terminate_process_tree "$command_pid"
+        break
+      fi
+    done
+  ) &
+  monitor_pid=$!
+
+  wait "$command_pid"
+  exit_code=$?
+  wait "$reader_pid" 2>/dev/null || true
+  kill "$monitor_pid" >/dev/null 2>&1 || true
+  wait "$monitor_pid" 2>/dev/null || true
+
+  if [ -e "$timeout_file" ]; then
+    rm -rf "$tmp_dir"
+    return 124
+  fi
+
+  rm -rf "$tmp_dir"
+  return "$exit_code"
+}
+
+# Function to run brew with retries and an inactivity watchdog.
+safe_brew() {
+  local max_attempts="${SETUP_PHP_BREW_RETRY_ATTEMPTS:-3}"
+  local attempt=1
+  local exit_code=0
+
+  if [ "${SETUP_PHP_BREW_WATCHDOG:-true}" = "false" ]; then
+    brew "$@"
+    return $?
+  fi
+
+  while [ "$attempt" -le "$max_attempts" ]; do
+    run_with_inactivity_watchdog brew "$@" && return 0
+    exit_code=$?
+
+    if [ "$attempt" -ge "$max_attempts" ]; then
+      return "$exit_code"
+    fi
+
+    printf "setup-php: retrying brew command (attempt %s/%s, exit %s)\n" "$((attempt + 1))" "$max_attempts" "$exit_code" >&2
+    sleep "$((attempt * 5))"
+    attempt=$((attempt + 1))
+  done
+
+  return "$exit_code"
+}
+
 # Function to add brew.
 add_brew() {
   brew_prefix="$(get_brew_prefix)"
@@ -74,6 +186,7 @@ configure_brew() {
   export HOMEBREW_NO_ENV_HINTS=1
   export HOMEBREW_NO_INSTALL_CLEANUP=1
   export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+  export HOMEBREW_DOWNLOAD_CONCURRENCY="${HOMEBREW_DOWNLOAD_CONCURRENCY:-6}"
   export brew_opts
   export brew_path
   export brew_path_dir