From 011982c501d5be45eefb21926dcde04748b6327a Mon Sep 17 00:00:00 2001 From: Ali Mohammad Pur Date: Thu, 24 Apr 2025 17:36:10 +0200 Subject: [PATCH] Meta: Make WPT.sh run nproc/2 testrunners in parallel This is the "intended" way of parallelism with wpt, but instead of requiring N different systems (or VMs), this does it all on one system with the power of namespaces. --- Meta/WPT.sh | 356 +++++++++++++++++++++++++++++++++++-- Meta/watch_wpt_progress.sh | 15 ++ 2 files changed, 358 insertions(+), 13 deletions(-) create mode 100644 Meta/watch_wpt_progress.sh diff --git a/Meta/WPT.sh b/Meta/WPT.sh index e8f3a912988..f43d6e900b9 100755 --- a/Meta/WPT.sh +++ b/Meta/WPT.sh @@ -16,6 +16,28 @@ BUILD_PRESET=${BUILD_PRESET:-default} BUILD_DIR=$(get_build_dir "$BUILD_PRESET") +: "${TRY_SHOW_LOGFILES_IN_TMUX:=false}" +: "${SHOW_LOGFILES:=true}" +: "${SHOW_PROGRESS:=true}" +: "${PARALLEL_INSTANCES:=1}" + +if "$SHOW_PROGRESS"; then + SHOW_LOGFILES=true + TRY_SHOW_LOGFILES_IN_TMUX=false +fi + +sudo_and_ask() { + local prompt + prompt="$1"; shift + if [ -z "$prompt" ]; then + prompt="Running '${*}' as root, please enter password for %p: " + else + prompt="$prompt; please enter password for %p: " + fi + + sudo --prompt="$prompt" "${@}" +} + default_binary_path() { if [ "$(uname -s)" = "Darwin" ]; then echo "${BUILD_DIR}/bin/Ladybird.app/Contents/MacOS" @@ -30,6 +52,25 @@ ladybird_git_hash() { popd > /dev/null } +run_dir_path() { + i="$1"; shift + local runpath="${BUILD_DIR}/wpt/run.$i" + echo "$runpath" +} + +ensure_run_dir() { + i="$1"; shift + local runpath + + runpath="$(run_dir_path "$i")" + if [ ! -d "$runpath" ]; then + mkdir -p "$runpath/upper" "$runpath/work" "$runpath/merged" + # shellcheck disable=SC2140 + sudo_and_ask "Mounting overlayfs on $runpath" mount -t overlay overlay -o lowerdir="${WPT_SOURCE_DIR}",upperdir="$runpath/upper",workdir="$runpath/work" "$runpath/merged" + fi + echo "$runpath/merged" +} + LADYBIRD_BINARY=${LADYBIRD_BINARY:-"$(default_binary_path)/Ladybird"} WEBDRIVER_BINARY=${WEBDRIVER_BINARY:-"$(default_binary_path)/WebDriver"} HEADLESS_BROWSER_BINARY=${HEADLESS_BROWSER_BINARY:-"$(default_binary_path)/headless-browser"} @@ -40,10 +81,8 @@ WPT_CERTIFICATES=( ) WPT_ARGS=( "--webdriver-binary=${WEBDRIVER_BINARY}" "--install-webdriver" - "--processes=${WPT_PROCESSES}" "--webdriver-arg=--force-cpu-painting" "--no-pause-after-test" - "-f" "${EXTRA_WPT_ARGS[@]}" ) @@ -62,6 +101,31 @@ print_help() { Fetch the given test file(s) from https://wpt.live/ and create an in-tree test and expectation files. list-tests: $NAME list-tests [PATHS..] List the tests in the given PATHS. + clean: $NAME clean + Clean up the extra resources and directories (if any leftover) created by this script. + + Env vars: + EXTRA_WPT_ARGS: Extra arguments for the wpt command, placed at the end; array, default empty + TRY_SHOW_LOGFILES_IN_TMUX: Whether to show split logs in tmux; true or false, default false + SHOW_LOGFILES: Whether to show logs at all; true or false, default true + SHOW_PROGRESS: Whether to show the progress of the tests, default true + implies SHOW_LOGFILES=true and TRY_SHOW_LOGFILES_IN_TMUX=false + + Options for this script: + --show-window + Disable headless mode + --debug-process PROC_NAME + Enable debugging for the PROC_NAME ladybird process + --parallel-instances N + Enable running in chunked mode with N parallel instances + N=0 to auto-enable if possible + N=1 to disable chunked mode (default) + N>1 to enable chunked mode with explicit process count + --log PATH + Alias for --log-raw PATH + --log-(raw|unittest|xunit|html|mach|tbpl|grouped|chromium|wptreport|wptscreenshot) PATH + Enable the given wpt log option with the given PATH + Examples: $NAME update @@ -72,6 +136,8 @@ print_help() { Run the Web Platform Tests in the 'css' and 'dom' directories and save the output to expectations.log. $NAME run --log-wptreport expectations.json --log-wptscreenshot expectations.db css dom Run the Web Platform Tests in the 'css' and 'dom' directories; save the output in wptreport format to expectations.json and save screenshots to expectations.db. + $NAME run --parallel-instances 0 --log-wptreport expectations.json --log-wptscreenshot expectations.db css dom + Run the Web Platform Tests in the 'css' and 'dom' directories in chunked mode; save the output in wptreport format to expectations.json and save screenshots to expectations.db. $NAME run --debug-process WebContent http://wpt.live/dom/historical.html Run the 'dom/historical.html' test, attaching the debugger to the WebContent process when the browser is launched. $NAME compare expectations.log @@ -104,14 +170,14 @@ set_logging_flags() [ -n "${2}" ] || usage; log_type="${1}" - log_name="$(absolutize_path "${2}")" + log_name="${2}" WPT_ARGS+=( "${log_type}=${log_name}" ) } headless=1 ARG=$1 -while [[ "$ARG" =~ ^(--show-window|--debug-process|(--log(-(raw|unittest|xunit|html|mach|tbpl|grouped|chromium|wptreport|wptscreenshot))?))$ ]]; do +while [[ "$ARG" =~ ^(--show-window|--debug-process|--parallel-instances|(--log(-(raw|unittest|xunit|html|mach|tbpl|grouped|chromium|wptreport|wptscreenshot))?))$ ]]; do case "$ARG" in --show-window) headless=0 @@ -125,6 +191,10 @@ while [[ "$ARG" =~ ^(--show-window|--debug-process|(--log(-(raw|unittest|xunit|h set_logging_flags "--log-raw" "${2}" shift ;; + --parallel-instances) + PARALLEL_INSTANCES="${2}" + shift + ;; *) set_logging_flags "${ARG}" "${2}" shift @@ -166,8 +236,7 @@ ensure_wpt_repository() { # Update hosts file if needed if [ "$(comm -13 <(sort -u /etc/hosts) <(./wpt make-hosts-file | sort -u) | wc -l)" -gt 0 ]; then - echo "Enter superuser password to append wpt hosts to /etc/hosts" - ./wpt make-hosts-file | sudo tee -a /etc/hosts + ./wpt make-hosts-file | sudo_and_ask "Appending wpt hosts to /etc/hosts" tee -a /etc/hosts fi popd > /dev/null } @@ -183,12 +252,270 @@ update_wpt() { popd > /dev/null } -execute_wpt() { - # Ensure open files limit is at least 1024, so the WPT runner does not run out of descriptors - if [ "$(ulimit -n)" -lt 1024 ]; then - ulimit -S -n 1024 +cleanup_run_infra() { + readarray -t pids < <(jobs -p) + for pid in "${pids[@]}"; do + if ps -p "$pid" > /dev/null; then + echo "Killing background process $pid" + kill -HUP "$pid" 2>/dev/null || true + fi + done + + readarray -t NSS < <(ip netns list 2>/dev/null | grep -E '^wptns[0-9]+' | awk '{print $1}') + if [ "${#NSS}" = 0 ]; then + return + fi + echo "Cleaning up namespaces: ${NSS[*]}" + for i in "${!NSS[@]}"; do + ns="${NSS[$i]}" + + echo "Cleaning up namespace: $ns" + + # Delete namespace + if sudo_and_ask "" ip netns list | grep -qw "$ns"; then + sudo_and_ask "Removing netns $ns" ip netns delete "$ns" || echo " failed to delete netns $ns" + fi + + # Remove hosts override + sudo_and_ask "Removing support files for netns $ns" rm -rf "/etc/netns/$ns" || echo " failed to delete /etc/netns/$ns" + done +} + +cleanup_run_dirs() { + readarray -t dirs < <(ls "${BUILD_DIR}/wpt" 2>/dev/null) + if [ "${#dirs}" = 0 ]; then + return fi + echo "Cleaning run dirs: ${dirs[*]}" + for dir in "${dirs[@]}"; do + mount_path="${BUILD_DIR}/wpt/$dir/merged" + for _ in $(seq 1 5); do + readarray -t pids_in_use < <(sudo_and_ask "" lsof "$mount_path" 2>/dev/null | cut -f2 -d' ') + [ "${#pids_in_use[@]}" = 0 ] && break + echo Trying to kill procs: "${pids_in_use[@]}" + kill -INT "${pids_in_use[@]}" 2>/dev/null || true + done + sudo_and_ask "" umount "$mount_path" || true + done + rm -fr "${BUILD_DIR}/wpt" + } +cleanup_merge_dirs_and_infra() { + cleanup_run_dirs + cleanup_run_infra +} +trap cleanup_merge_dirs_and_infra EXIT INT TERM + +make_instances() { + if [ "${PARALLEL_INSTANCES}" = 1 ]; then + echo 1 + return + fi + + if ! command -v ip &>/dev/null; then + echo "the 'ip' command is required to run WPT in chunked mode" >&2 + echo 1 + return + fi + + if ! sudo_and_ask "Making test netns 'testns'" ip netns add testns; then + echo "ip netns failed, chunked mode not available" >&2 + echo 1 + return + fi + sudo_and_ask "Cleaning up test netns 'testns'" ip netns delete testns + + explicit_count="${PARALLEL_INSTANCES}" + + local total_cores count ns + total_cores=$(nproc) + count=$(( total_cores / 2 )) + (( count < 1 )) && count=1 + + if (( explicit_count > 0 )); then + count="$explicit_count" + fi + + for i in $(seq 0 $((count - 1))); do + ns="wptns$i" + + # Create namespace + sudo_and_ask "" ip netns add "$ns" + sudo_and_ask "" ip netns exec "$ns" ip link set lo up + + # Setup DNS and hosts (we've messed with it before getting here) + sudo_and_ask "" mkdir -p "/etc/netns/$ns" + sudo_and_ask "" cp /etc/hosts "/etc/netns/$ns/hosts" + done + + echo "$count" +} + +instance_run() { + local idx="$1"; shift + local rundir="$1"; shift + local ns="wptns$idx" + if sudo_and_ask "" ip netns list | grep -qw "$ns"; then + ( + cd "$rundir" + sudo_and_ask "" ip netns exec "$ns" sudo -u "$USER" -- env "PATH=$PATH" "$@" + ) + else + echo " netns $ns not found, running in the host namespace" + ( + cd "${WPT_SOURCE_DIR}" + "$@" + ) + fi +} + +show_files() { + if ! "$SHOW_LOGFILES"; then + return + fi + local files=("$@") + if ! command -v tmux &>/dev/null || ! "$TRY_SHOW_LOGFILES_IN_TMUX"; then + if "$TRY_SHOW_LOGFILES_IN_TMUX"; then + echo "tmux is not available, falling back to tail" + fi + if "$SHOW_PROGRESS"; then + bash "${DIR}/watch_wpt_progress.sh" "${files[@]}" & + else + tail -f "${files[@]}" & + fi + PID=$! + for pid in $(jobs -p | grep -v $PID); do + # shellcheck disable=SC2009 + ps | grep -q "$pid" && wait "$pid" + done + kill -HUP $PID + else + tmux new-session -d + + tmux send-keys "less +F ${files[0]}" C-m + + for ((i = 1; i < ${#files[@]}; i++)); do + if (( i % 2 == 1 )); then + tmux split-window -h "less +F ${files[i]}" + else + tmux split-window -v "less +F ${files[i]}" + fi + tmux select-layout tiled > /dev/null + done + + tmux attach + fi +} + +copy_results_to() { + local target="$1"; shift + local runcount="$1"; shift + mkdir -p "$target" + for i in $(seq 0 $((runcount - 1))); do + for f in "$(run_dir_path "$i")/upper"/*; do + cp -r "$f" "$target/$(basename "$f").run_$i" + done + done +} + +show_summary() { + local logs=("$@") + local total_tests=0 + local expected=0 + local skipped=0 + local errored=0 + local subtest_issues=0 + local max_time=0 + + for out_file in "${logs[@]}"; do + # wpt puts random garbage in the output, strip those (as they're all nonprint) + mapfile -t lines < <(grep -A4 -aE 'Ran [0-9]+ tests finished in' "$out_file" \ + | iconv -f utf-8 -t ascii//TRANSLIT 2>/dev/null \ + | sed 's/[^[:print:]]//g') + + for line in "${lines[@]}"; do + if [[ $line =~ Ran[[:space:]]([0-9]+)[[:space:]]tests[[:space:]]finished[[:space:]]in[[:space:]]([0-9.]+) ]]; then + (( total_tests += BASH_REMATCH[1] )) + time=${BASH_REMATCH[2]} + [[ $(echo "$time > $max_time" | bc -l) == 1 ]] && max_time=$time + elif [[ $line =~ ([0-9]+)[[:space:]]ran[[:space:]]as[[:space:]]expected ]]; then + (( expected += BASH_REMATCH[1] )) + elif [[ $line =~ ([0-9]+)[[:space:]]tests[[:space:]]skipped ]]; then + (( skipped += BASH_REMATCH[1] )) + elif [[ $line =~ ([0-9]+)[[:space:]]tests[[:space:]](crashed|timed[[:space:]]out|had[[:space:]]errors)[[:space:]]unexpectedly ]]; then + (( errored += BASH_REMATCH[1] )) + elif [[ $line =~ ([0-9]+)[[:space:]]tests[[:space:]]had[[:space:]]unexpected[[:space:]]subtest[[:space:]]results ]]; then + (( subtest_issues += BASH_REMATCH[1] )) + fi + done + done + + echo "Total tests run: $total_tests" + echo "Ran as expected: $expected" + echo "Skipped: $skipped" + echo "Errored unexpectedly: $errored" + echo "Unexpected subtest results: $subtest_issues" + echo "Longest run time: ${max_time}s" +} + +run_wpt_chunked() { + local procs concurrency + procs=$(make_instances) + + # Ensure open files limit is at least 1024, so the WPT runner does not run out of descriptors + if [ "$(ulimit -n)" -lt $((1024 * procs)) ]; then + ulimit -S -n $((1024 * procs)) + fi + + if [ "$procs" -le 1 ]; then + command=(./wpt run -f --processes="${WPT_PROCESSES}" "$@") + echo "${command[@]}" + "${command[@]}" + return + fi + + concurrency=$(( $(nproc) * 2 / procs )) + LADYBIRD_GIT_VERSION="$(ladybird_git_hash)" + + echo "Preparing the venv setup..." + base_venv="${BUILD_DIR}/wpt-prep/_venv" + ./wpt --venv "$base_venv" run "${WPT_ARGS[@]}" ladybird THIS_TEST_CANNOT_POSSIBLY_EXIST || true + + echo "Launching $procs chunked instances (concurrency=$concurrency each)" + export LADYBIRD_GIT_VERSION + local logs=() + + for i in $(seq 0 $((procs - 1))); do + local rundir runpath logpath + rundir="$(ensure_run_dir "$i")" + runpath="$(run_dir_path "$i")" + logpath="$runpath/upper/run.logs" + echo "rundir at $rundir, logs in $logpath" + touch "$logpath" + logs+=("$logpath") + + cp -r "$base_venv" "${runpath}/_venv" + + command=(./wpt --venv "${runpath}/_venv" \ + run \ + --this-chunk="$((i + 1))" \ + --total-chunks="$procs" \ + --chunk-type=hash \ + -f \ + --processes="$concurrency" \ + "$@") + echo "[INSTANCE $i / ns wptns$i] LADYBIRD_GIT_VERSION=$LADYBIRD_GIT_VERSION ${command[*]}" + instance_run "$i" "$rundir" script -q "$logpath" -c "$(printf "%q " "${command[@]}")" &>/dev/null & + done + + show_files "${logs[@]}" + wait + + copy_results_to "${BUILD_DIR}/wpt-run-$(date +%s)" "$procs" + show_summary "${logs[@]}" +} + +execute_wpt() { pushd "${WPT_SOURCE_DIR}" > /dev/null for certificate_path in "${WPT_CERTIFICATES[@]}"; do if [ ! -f "${certificate_path}" ]; then @@ -198,8 +525,7 @@ execute_wpt() { WPT_ARGS+=( "--webdriver-arg=--certificate=${certificate_path}" ) done construct_test_list "${@}" - echo LADYBIRD_GIT_VERSION="$(ladybird_git_hash)" ./wpt run "${WPT_ARGS[@]}" ladybird "${TEST_LIST[@]}" - LADYBIRD_GIT_VERSION="$(ladybird_git_hash)" ./wpt run "${WPT_ARGS[@]}" ladybird "${TEST_LIST[@]}" + run_wpt_chunked "${WPT_ARGS[@]}" ladybird "${TEST_LIST[@]}" popd > /dev/null } @@ -280,7 +606,7 @@ compare_wpt() { rm -rf "${METADATA_DIR}" } -if [[ "$CMD" =~ ^(update|run|serve|compare|import|list-tests)$ ]]; then +if [[ "$CMD" =~ ^(update|clean|run|serve|compare|import|list-tests)$ ]]; then case "$CMD" in update) update_wpt @@ -288,6 +614,10 @@ if [[ "$CMD" =~ ^(update|run|serve|compare|import|list-tests)$ ]]; then run) run_wpt "${@}" ;; + clean) + cleanup_run_infra + cleanup_run_dirs true + ;; serve) serve_wpt ;; diff --git a/Meta/watch_wpt_progress.sh b/Meta/watch_wpt_progress.sh new file mode 100644 index 00000000000..84bc48fb319 --- /dev/null +++ b/Meta/watch_wpt_progress.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +files=("${@}") + +while true; do + out=$( + for file in "${files[@]}"; do + echo -n "$file:" + { grep -aohE "^\s*\[[0-9]+/[0-9]+\]" "$file" || echo ' [not started yet]'; } | tail -n1 + done + ) + clear + echo "$out" + sleep 1 +done