Fix Homebrew timeout cleanup and retry propagation

This commit is contained in:
Shivam Mathur
2026-09-11 14:20:07 +05:30
parent 250924180e
commit 0c6410c670
2 changed files with 61 additions and 29 deletions
+10 -4
View File
@@ -183,6 +183,7 @@ setup_cached_versions() {
# Function to setup PHP 5.6 and newer using Homebrew. # Function to setup PHP 5.6 and newer using Homebrew.
add_php() { add_php() {
local exit_code
action=$1 action=$1
existing_version=$2 existing_version=$2
suffix="$(get_php_formula_suffix)" suffix="$(get_php_formula_suffix)"
@@ -198,14 +199,19 @@ add_php() {
fi fi
if [[ "$existing_version" != "false" && -z "$suffix" ]]; then if [[ "$existing_version" != "false" && -z "$suffix" ]]; then
if [ "$action" = "upgrade" ]; then if [ "$action" = "upgrade" ]; then
safe_brew install --only-dependencies "$php_formula" safe_brew install --only-dependencies "$php_formula" || return $?
safe_brew upgrade -f --overwrite "$php_formula" safe_brew upgrade -f --overwrite "$php_formula" || return $?
else else
brew unlink "$php_keg" brew unlink "$php_keg"
fi fi
else else
safe_brew install --only-dependencies "$php_formula" safe_brew install --only-dependencies "$php_formula" || return $?
safe_brew install --skip-link -f --overwrite "$php_formula" 2>/dev/null || safe_brew upgrade -f --overwrite "$php_formula" safe_brew install --skip-link -f --overwrite "$php_formula" 2>/dev/null || {
exit_code=$?
# A timeout has exhausted its retries; do not start another build via upgrade.
[ "$exit_code" -ne 124 ] || return "$exit_code"
safe_brew upgrade -f --overwrite "$php_formula" || return $?
}
fi fi
brew link --force --overwrite "$php_keg" || (sudo chown -R "$(id -un)":"$(id -gn)" "$brew_prefix" && brew link --force --overwrite "$php_keg") brew link --force --overwrite "$php_keg" || (sudo chown -R "$(id -un)":"$(id -gn)" "$brew_prefix" && brew link --force --overwrite "$php_keg")
} }
+51 -25
View File
@@ -55,25 +55,45 @@ get_file_mtime() {
fi fi
} }
# Function to terminate a process and its direct children. # Function to list descendants before their parents, including separate sessions.
terminate_process_tree() { get_process_tree() {
local pid=$1 local pid=$1
local children child local children child
children=$(pgrep -P "$pid" 2>/dev/null || true) children=$(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
for child in $children; do for child in $children; do
terminate_process_tree "$child" get_process_tree "$child"
done
echo "$pid"
}
# Function to detect Homebrew's source-build worker, even with buffered output.
is_brew_building_from_source() {
local pid
for pid in $(get_process_tree "$1"); do
if ps -ww -p "$pid" -o command= 2>/dev/null | grep -qE '/Homebrew/build[.]rb([[:space:]]|$)'; then
return 0
fi
done
return 1
}
# Function to terminate the entire tree captured before any parents can exit.
terminate_process_tree() {
local pids pid
pids=$(get_process_tree "$1")
for pid in $pids; do
kill -TERM "$pid" >/dev/null 2>&1 || true
done done
sleep 2 sleep 2
kill -KILL "$pid" >/dev/null 2>&1 || true for pid in $pids; do
for child in $children; do kill -KILL "$pid" >/dev/null 2>&1 || true
terminate_process_tree "$child"
done done
} }
# Function to run a command with an inactivity watchdog. # Function to run a command with an inactivity watchdog.
run_with_inactivity_watchdog() { run_with_inactivity_watchdog() {
local timeout_secs="${SETUP_PHP_BREW_INACTIVITY_TIMEOUT:-180}" local timeout_secs="${SETUP_PHP_BREW_INACTIVITY_TIMEOUT:-180}"
local source_timeout_secs="${SETUP_PHP_BREW_SOURCE_INACTIVITY_TIMEOUT:-1800}"
local poll_secs="${SETUP_PHP_BREW_WATCHDOG_POLL:-5}" local poll_secs="${SETUP_PHP_BREW_WATCHDOG_POLL:-5}"
local tmp_dir stdout_fifo stderr_fifo stdout_log stderr_log timeout_file local tmp_dir stdout_fifo stderr_fifo stdout_log stderr_log timeout_file
local command_pid stdout_reader_pid stderr_reader_pid monitor_pid exit_code local command_pid stdout_reader_pid stderr_reader_pid monitor_pid exit_code
@@ -93,36 +113,39 @@ run_with_inactivity_watchdog() {
("$@" >"$stdout_fifo" 2>"$stderr_fifo") & ("$@" >"$stdout_fifo" 2>"$stderr_fifo") &
command_pid=$! command_pid=$!
( tee "$stdout_log" <"$stdout_fifo" &
while IFS= read -r line || [ -n "$line" ]; do
printf '%s\n' "$line"
printf '%s\n' "$line" >>"$stdout_log"
done <"$stdout_fifo"
) &
stdout_reader_pid=$! stdout_reader_pid=$!
( tee "$stderr_log" <"$stderr_fifo" >&2 &
while IFS= read -r line || [ -n "$line" ]; do
printf '%s\n' "$line" >&2
printf '%s\n' "$line" >>"$stderr_log"
done <"$stderr_fifo"
) &
stderr_reader_pid=$! stderr_reader_pid=$!
( (
local last_activity current_activity current_err_activity now local last_activity current_activity current_err_activity now
local building_from_source=false was_building_from_source=false active_timeout_secs
last_activity=$(get_file_mtime "$stdout_log") last_activity=$(get_file_mtime "$stdout_log")
current_err_activity=$(get_file_mtime "$stderr_log") current_err_activity=$(get_file_mtime "$stderr_log")
[ "$current_err_activity" -gt "$last_activity" ] && last_activity="$current_err_activity" [ "$current_err_activity" -gt "$last_activity" ] && last_activity="$current_err_activity"
while kill -0 "$command_pid" >/dev/null 2>&1; do while kill -0 "$command_pid" >/dev/null 2>&1; do
sleep "$poll_secs" sleep "$poll_secs"
kill -0 "$command_pid" >/dev/null 2>&1 || break
now=$(date +%s)
active_timeout_secs="$timeout_secs"
building_from_source=false
if is_brew_building_from_source "$command_pid"; then
building_from_source=true
active_timeout_secs="$source_timeout_secs"
fi
if [ "$building_from_source" != "$was_building_from_source" ]; then
last_activity="$now"
was_building_from_source="$building_from_source"
fi
current_activity=$(get_file_mtime "$stdout_log") current_activity=$(get_file_mtime "$stdout_log")
[ "$current_activity" -gt "$last_activity" ] && last_activity="$current_activity" [ "$current_activity" -gt "$last_activity" ] && last_activity="$current_activity"
current_err_activity=$(get_file_mtime "$stderr_log") current_err_activity=$(get_file_mtime "$stderr_log")
[ "$current_err_activity" -gt "$last_activity" ] && last_activity="$current_err_activity" [ "$current_err_activity" -gt "$last_activity" ] && last_activity="$current_err_activity"
now=$(date +%s) now=$(date +%s)
if [ $((now - last_activity)) -ge "$timeout_secs" ]; then if [ $((now - last_activity)) -ge "$active_timeout_secs" ]; then
printf "\nsetup-php: brew produced no output for %ss; terminating and retrying...\n" "$timeout_secs" >&2 printf "\nsetup-php: brew produced no output for %ss; terminating...\n" "$active_timeout_secs" >&2
: >"$timeout_file" : >"$timeout_file"
terminate_process_tree "$command_pid" terminate_process_tree "$command_pid"
break break
@@ -131,12 +154,15 @@ run_with_inactivity_watchdog() {
) & ) &
monitor_pid=$! monitor_pid=$!
wait "$command_pid" exit_code=0
exit_code=$? wait "$command_pid" || exit_code=$?
# Let timeout cleanup finish killing source-build descendants before retrying.
if [ ! -e "$timeout_file" ]; then
kill "$monitor_pid" >/dev/null 2>&1 || true
fi
wait "$monitor_pid" 2>/dev/null || true
wait "$stdout_reader_pid" 2>/dev/null || true wait "$stdout_reader_pid" 2>/dev/null || true
wait "$stderr_reader_pid" 2>/dev/null || true wait "$stderr_reader_pid" 2>/dev/null || true
kill "$monitor_pid" >/dev/null 2>&1 || true
wait "$monitor_pid" 2>/dev/null || true
if [ -e "$timeout_file" ]; then if [ -e "$timeout_file" ]; then
rm -rf "$tmp_dir" rm -rf "$tmp_dir"