From 1e30fd5c6a8d68cf1696240659b9c3623fcac904 Mon Sep 17 00:00:00 2001 From: Tim Ledbetter Date: Tue, 15 Jul 2025 20:14:07 +0100 Subject: [PATCH] Meta: Add `WPT.sh bisect` for finding WPT regressions automatically --- Meta/WPT.sh | 105 +++++++++++++++++++++++++++++++++++++- Meta/wpt-bisect-helper.sh | 42 +++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100755 Meta/wpt-bisect-helper.sh diff --git a/Meta/WPT.sh b/Meta/WPT.sh index 8446520c0ce..5c7a9906690 100755 --- a/Meta/WPT.sh +++ b/Meta/WPT.sh @@ -16,6 +16,8 @@ BUILD_PRESET=${BUILD_PRESET:-default} BUILD_DIR=$(get_build_dir "$BUILD_PRESET") +TMPDIR=${TMPDIR:-/tmp} + : "${TRY_SHOW_LOGFILES_IN_TMUX:=false}" : "${SHOW_LOGFILES:=true}" : "${SHOW_PROGRESS:=true}" @@ -107,6 +109,8 @@ print_help() { List the tests in the given PATHS. clean: $NAME clean Clean up the extra resources and directories (if any leftover) created by this script. + bisect: $NAME bisect BAD_COMMIT GOOD_COMMIT [TESTS...] + Find the first commit where a given set of tests produce unexpected results. Env vars: EXTRA_WPT_ARGS: Extra arguments for the wpt command, placed at the end; array, default empty @@ -245,7 +249,7 @@ ensure_wpt_repository() { } build_ladybird_and_webdriver() { - "${DIR}"/ladybird.py build WebDriver + "${LADYBIRD_SOURCE_DIR}"/Meta/ladybird.py build WebDriver } update_wpt() { @@ -565,6 +569,96 @@ serve_wpt() popd > /dev/null } +cleanup_bisect() +{ + local temp_file_directory="$1"; shift + if [ -d "${temp_file_directory}" ]; then + echo "Removing temp file directory: ${temp_file_directory}" + rm -rf "${temp_file_directory}" + fi + + git bisect reset +} + +bisect_wpt() +{ + if ! git diff-index --quiet HEAD --; then + echo "You have uncommitted changes, please commit or stash them before bisecting." + exit 1 + fi + + local bad="$1"; shift + local good="$1"; shift + # Commits from before ladybird.py was added don't currently work with this script + OLDEST_COMMIT_ALLOWED="061a7f766ce" + + if ! git rev-parse --verify "${bad}" >/dev/null 2>&1; then + echo "Bad commit '${bad}' not found." + exit 1 + fi + + if ! git rev-parse --verify "${good}" >/dev/null 2>&1; then + echo "Good commit '${good}' not found." + exit 1 + fi + + if ! git merge-base --is-ancestor ${OLDEST_COMMIT_ALLOWED} "${good}"; then + echo "Commits older than ${OLDEST_COMMIT_ALLOWED} aren't allowed (because ladybird.py is required)." + exit 1 + fi + + if [ "${good}" = "${bad}" ]; then + echo "The good commit and the bad commit must be different." + exit 1 + fi + + if ! git merge-base --is-ancestor "${good}" "${bad}" ; then + echo "The good commit must be older than the bad commit." + exit 1 + fi + + ensure_wpt_repository + construct_test_list "${@}" + + pushd "${LADYBIRD_SOURCE_DIR}" > /dev/null + local temp_file_directory_base + temp_file_directory_base="$(mktemp -p "${TMPDIR}" -d "wpt-bisect-helper-XXXXXX")" + mkdir "${temp_file_directory_base}/Meta" + local baseline_log_file + baseline_log_file="$(mktemp -p "${temp_file_directory_base}" -u "XXXXXX.log")" + local current_branch_or_commit + current_branch_or_commit="$(git branch --show-current 2> /dev/null)" + if [ -z "${current_branch_or_commit}" ]; then + current_branch_or_commit="$(git rev-parse HEAD)" + fi + + # We create the baseline log file against the bad commit bcause building it may be significantly faster if the + # good commit is significantly older than the bad commit. + git checkout "${bad}" 2> /dev/null + trap 'git checkout "${current_branch_or_commit}" 2> /dev/null' EXIT INT TERM + echo "Generating baseline log file at \"${baseline_log_file}\"" + $0 run --log "${baseline_log_file}" "${@}" || true + trap - EXIT INT TERM + git checkout "${current_branch_or_commit}" 2> /dev/null + + # We need to copy scripts that will run during bisection to ensure that we will always have the latest version. + required_build_files=( + "shell_include.sh" + "wpt-bisect-helper.sh" + ) + for file in "${required_build_files[@]}"; do + cp "${LADYBIRD_SOURCE_DIR}/Meta/${file}" "${temp_file_directory_base}/Meta/${file}" + done + cp "$0" "${temp_file_directory_base}/Meta/WPT.sh" + + git bisect start "${bad}" "${good}" + trap cleanup_bisect INT TERM + git bisect run "${temp_file_directory_base}/Meta/wpt-bisect-helper.sh" "${baseline_log_file}" "${@}" || true + trap - INT TERM + cleanup_bisect "${temp_file_directory_base}" + popd > /dev/null +} + list_tests_wpt() { ensure_wpt_repository @@ -643,7 +737,7 @@ compare_wpt() { rm -rf "${METADATA_DIR}" } -if [[ "$CMD" =~ ^(update|clean|run|serve|compare|import|list-tests)$ ]]; then +if [[ "$CMD" =~ ^(update|clean|run|serve|bisect|compare|import|list-tests)$ ]]; then case "$CMD" in update) update_wpt @@ -658,6 +752,13 @@ if [[ "$CMD" =~ ^(update|clean|run|serve|compare|import|list-tests)$ ]]; then serve) serve_wpt ;; + bisect) + if [ $# -lt 3 ]; then + echo "Usage: $0 bisect [test paths...]" + usage + fi + bisect_wpt "${@}" + ;; import) while [[ "$1" =~ ^--(force|wpt-base-url)$ ]]; do if [ "$1" = "--wpt-base-url" ]; then diff --git a/Meta/wpt-bisect-helper.sh b/Meta/wpt-bisect-helper.sh new file mode 100755 index 00000000000..966f08d89c7 --- /dev/null +++ b/Meta/wpt-bisect-helper.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# An exit code of 255 tells git to stop bisecting immediately. +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [test paths...]" + exit 255 +fi + +baseline_log_file="$1" +if [ ! -f "${baseline_log_file}" ]; then + echo "Baseline log file '${baseline_log_file}' does not exist." + exit 255 +fi +shift + +WPT_SCRIPT_PATH=${WPT_SCRIPT_PATH:-"$(dirname "$0")/WPT.sh"} + +pushd "${LADYBIRD_SOURCE_DIR}" > /dev/null || exit 255 + trap "exit 255" INT TERM + if ! ./Meta/ladybird.py rebuild WebDriver; then + # When going back over many commits rebuilds may be flaky. Let's try again before skipping this commit. + if ! ./Meta/ladybird.py rebuild WebDriver; then + echo "Build failed twice in a row, skipping this commit." + exit 125 + fi + fi + + current_commit="$(git rev-parse HEAD)" + + # If comparing to the baseline has no unexpected results this commit is bad, because we create the basseline from + # the given bad commit. + if "${WPT_SCRIPT_PATH}" compare "${baseline_log_file}" "${@}"; then + echo "Commit ${current_commit} is bad." + exit 1 + fi + trap - INT TERM +popd > /dev/null || exit 255 + +echo "Commit ${current_commit} is good." +exit 0