diff --git a/Libraries/LibWeb/Internals/Internals.cpp b/Libraries/LibWeb/Internals/Internals.cpp index e6594e40552..f53e88f9706 100644 --- a/Libraries/LibWeb/Internals/Internals.cpp +++ b/Libraries/LibWeb/Internals/Internals.cpp @@ -81,6 +81,26 @@ WebIDL::ExceptionOr Internals::load_reference_test_metadata() metadata.set("match_references"sv, TRY(collect_references("match"sv))); metadata.set("mismatch_references"sv, TRY(collect_references("mismatch"sv))); + // Collect all values. + JsonArray fuzzy_configurations; + auto fuzzy_nodes = TRY(document->query_selector_all("meta[name=fuzzy]"sv)); + for (size_t i = 0; i < fuzzy_nodes->length(); ++i) { + auto const* fuzzy_node = fuzzy_nodes->item(i); + auto content = as(fuzzy_node)->get_attribute_value(HTML::AttributeNames::content); + + JsonObject fuzzy_configuration; + if (content.contains(':')) { + auto content_parts = MUST(content.split_limit(':', 2)); + auto reference_url = document->encoding_parse_url(content_parts[0]); + fuzzy_configuration.set("reference"sv, reference_url->to_string()); + content = content_parts[1]; + } + fuzzy_configuration.set("content"sv, content); + + fuzzy_configurations.must_append(fuzzy_configuration); + } + metadata.set("fuzzy"sv, fuzzy_configurations); + page.client().page_did_receive_reference_test_metadata(metadata); return {}; } diff --git a/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html new file mode 100644 index 00000000000..e50fc11ef6e --- /dev/null +++ b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html @@ -0,0 +1,9 @@ + + +
diff --git a/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/green.html b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/green.html new file mode 100644 index 00000000000..0c9676a7fdc --- /dev/null +++ b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/green.html @@ -0,0 +1,6 @@ + diff --git a/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/red.html b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/red.html new file mode 100644 index 00000000000..2b677e00634 --- /dev/null +++ b/Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/red.html @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_1.html b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_1.html new file mode 100644 index 00000000000..458f57f7482 --- /dev/null +++ b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_1.html @@ -0,0 +1,12 @@ + + + + +
+ diff --git a/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_no_differences.html b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_no_differences.html new file mode 100644 index 00000000000..b1546265b1f --- /dev/null +++ b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_no_differences.html @@ -0,0 +1,13 @@ + + + + + +
diff --git a/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_match.html b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_match.html new file mode 100644 index 00000000000..8ce7cf03474 --- /dev/null +++ b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_match.html @@ -0,0 +1,5 @@ +rel=match that should pass + + \ No newline at end of file diff --git a/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_mismatch.html b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_mismatch.html new file mode 100644 index 00000000000..92e8875e3b6 --- /dev/null +++ b/Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_mismatch.html @@ -0,0 +1,5 @@ +rel=mismatch that should pass + + \ No newline at end of file diff --git a/Tests/LibWeb/test-web/CMakeLists.txt b/Tests/LibWeb/test-web/CMakeLists.txt index 6298736782b..0e0f62ac9b9 100644 --- a/Tests/LibWeb/test-web/CMakeLists.txt +++ b/Tests/LibWeb/test-web/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES Application.cpp Fixture.cpp + Fuzzy.cpp TestWebView.cpp main.cpp ) diff --git a/Tests/LibWeb/test-web/Fuzzy.cpp b/Tests/LibWeb/test-web/Fuzzy.cpp new file mode 100644 index 00000000000..fdf8d16232a --- /dev/null +++ b/Tests/LibWeb/test-web/Fuzzy.cpp @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2025, Jelle Raaijmakers + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "Fuzzy.h" + +#include +#include +#include + +namespace TestWeb { + +// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching +bool fuzzy_screenshot_match(URL::URL const& reference, Gfx::Bitmap const& bitmap_a, Gfx::Bitmap const& bitmap_b, + Vector const& fuzzy_matches) +{ + // If the bitmaps are identical, we don't perform fuzzy matching. + auto diff = bitmap_a.diff(bitmap_b); + if (diff.identical) + return true; + + // Find a single fuzzy config to apply. + auto fuzzy_match = fuzzy_matches.first_matching([&reference](FuzzyMatch const& fuzzy_match) { + if (fuzzy_match.reference.has_value()) + return fuzzy_match.reference.value().equals(reference); + return true; + }); + if (!fuzzy_match.has_value()) + return diff.identical; + + // Apply fuzzy matching. + auto color_error_matches = fuzzy_match->color_value_error.contains(diff.maximum_error); + if (!color_error_matches) + warnln("Fuzzy mismatch: maximum error {} is outside {}", diff.maximum_error, fuzzy_match->color_value_error); + + auto pixel_error_matches = fuzzy_match->pixel_error_count.contains(diff.pixel_error_count); + if (!pixel_error_matches) + warnln("Fuzzy mismatch: pixel error count {} is outside {}", diff.pixel_error_count, fuzzy_match->pixel_error_count); + + return color_error_matches && pixel_error_matches; +} + +// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching +ErrorOr parse_fuzzy_range(String const& fuzzy_range) +{ + auto range_parts = MUST(fuzzy_range.split('-')); + if (range_parts.is_empty() || range_parts.size() > 2) + return Error::from_string_view("Invalid fuzzy range format"sv); + + auto parse_value = [](String const& value) -> ErrorOr { + auto maybe_value = value.to_number(); + if (!maybe_value.has_value()) + return Error::from_string_view("Fuzzy range value is not a valid integer"sv); + return maybe_value.release_value(); + }; + auto minimum_value = TRY(parse_value(range_parts[0])); + auto maximum_value = minimum_value; + if (range_parts.size() == 2) + maximum_value = TRY(parse_value(range_parts[1])); + + if (minimum_value > maximum_value) + return Error::from_string_view("Fuzzy range minimum is higher than its maximum"sv); + + return FuzzyRange { .minimum_value = minimum_value, .maximum_value = maximum_value }; +} + +// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching +ErrorOr parse_fuzzy_match(Optional reference, String const& content) +{ + // Match configuration values. Two formats are supported: + // * Named: "maxDifference=(#X-)#Y;totalPixels=(#X-)#Y" + // * Unnamed: "(#X-)#Y;(#X-)#Y" (maxDifference and totalPixels are assumed in this order) + auto config_parts = MUST(content.split(';')); + if (config_parts.size() != 2) + return Error::from_string_view("Fuzzy configuration must have exactly two parameters"sv); + + Optional color_value_error; + Optional pixel_error_count; + for (auto [i, config_part] : enumerate(config_parts)) { + auto named_parts = MUST(config_part.split_limit('=', 2)); + if (named_parts.is_empty()) + return Error::from_string_view("Fuzzy configuration value cannot be empty"sv); + + if (named_parts.size() == 2) { + if (named_parts[0] == "maxDifference"sv && !color_value_error.has_value()) + color_value_error = TRY(parse_fuzzy_range(named_parts[1])); + else if (named_parts[0] == "totalPixels"sv && !pixel_error_count.has_value()) + pixel_error_count = TRY(parse_fuzzy_range(named_parts[1])); + else + return Error::from_string_view("Invalid fuzzy configuration parameter"sv); + } else { + if (i == 0 && !color_value_error.has_value()) + color_value_error = TRY(parse_fuzzy_range(config_part)); + else if (i == 1 && !pixel_error_count.has_value()) + pixel_error_count = TRY(parse_fuzzy_range(config_part)); + else + return Error::from_string_view("Invalid fuzzy configuration parameter"sv); + } + } + + return FuzzyMatch { + .reference = reference.map([](URL::URL const& reference) { return reference; }), + .color_value_error = color_value_error.release_value(), + .pixel_error_count = pixel_error_count.release_value(), + }; +} + +} diff --git a/Tests/LibWeb/test-web/Fuzzy.h b/Tests/LibWeb/test-web/Fuzzy.h new file mode 100644 index 00000000000..4b90215d133 --- /dev/null +++ b/Tests/LibWeb/test-web/Fuzzy.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2025, Jelle Raaijmakers + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace TestWeb { + +struct FuzzyRange { + u64 minimum_value; + u64 maximum_value; + + bool contains(u64 value) const { return value >= minimum_value && value <= maximum_value; } +}; + +struct FuzzyMatch { + Optional reference; + FuzzyRange color_value_error; + FuzzyRange pixel_error_count; +}; + +bool fuzzy_screenshot_match(URL::URL const& reference, Gfx::Bitmap const&, Gfx::Bitmap const&, Vector const&); +ErrorOr parse_fuzzy_match(Optional reference, String const&); +ErrorOr parse_fuzzy_range(String const&); + +} + +namespace AK { + +template<> +struct Formatter : Formatter { + ErrorOr format(FormatBuilder& builder, TestWeb::FuzzyRange const& value) + { + return Formatter::format(builder, "FuzzyRange [{}-{}]"sv, value.minimum_value, value.maximum_value); + } +}; + +} diff --git a/Tests/LibWeb/test-web/TestWeb.h b/Tests/LibWeb/test-web/TestWeb.h index a253cf0d65f..db3edd93d7a 100644 --- a/Tests/LibWeb/test-web/TestWeb.h +++ b/Tests/LibWeb/test-web/TestWeb.h @@ -1,15 +1,19 @@ /* * Copyright (c) 2024-2025, Tim Flynn + * Copyright (c) 2025, Jelle Raaijmakers * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once +#include "Fuzzy.h" + #include #include #include #include +#include #include #include @@ -51,6 +55,7 @@ struct Test { bool did_finish_loading { false }; Optional ref_test_expectation_type {}; + Vector fuzzy_matches {}; RefPtr actual_screenshot {}; RefPtr expectation_screenshot {}; diff --git a/Tests/LibWeb/test-web/main.cpp b/Tests/LibWeb/test-web/main.cpp index 35664bbe92e..30486cf60e4 100644 --- a/Tests/LibWeb/test-web/main.cpp +++ b/Tests/LibWeb/test-web/main.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -375,10 +374,11 @@ static void run_ref_test(TestWebView& view, Test& test, URL::URL const& url, int view.on_test_complete({ test, TestResult::Timeout }); }); - auto handle_completed_test = [&test, url]() -> ErrorOr { + auto handle_completed_test = [&view, &test, url]() -> ErrorOr { VERIFY(test.ref_test_expectation_type.has_value()); auto should_match = test.ref_test_expectation_type == RefTestExpectationType::Match; - auto screenshot_matches = test.actual_screenshot->diff(*test.expectation_screenshot).identical; + auto screenshot_matches = fuzzy_screenshot_match( + view.url(), *test.actual_screenshot, *test.expectation_screenshot, test.fuzzy_matches); if (should_match == screenshot_matches) return TestResult::Pass; @@ -450,6 +450,27 @@ static void run_ref_test(TestWebView& view, Test& test, URL::URL const& url, int auto mismatch_references = metadata_object.get_array("mismatch_references"sv); VERIFY(!match_references->is_empty() || !mismatch_references->is_empty()); + // Read fuzzy configurations. + test.fuzzy_matches.clear_with_capacity(); + auto fuzzy_values = metadata_object.get_array("fuzzy"sv); + for (size_t i = 0; i < fuzzy_values->size(); ++i) { + auto fuzzy_configuration = fuzzy_values->at(i).as_object(); + + Optional reference_url; + auto reference = fuzzy_configuration.get_string("reference"sv); + if (reference.has_value()) + reference_url = URL::Parser::basic_parse(reference.release_value()); + + auto content = fuzzy_configuration.get_string("content"sv).release_value(); + auto fuzzy_match_or_error = parse_fuzzy_match(reference_url, content); + if (fuzzy_match_or_error.is_error()) { + warnln("Failed to parse fuzzy configuration '{}' (reference: {})", content, reference_url); + continue; + } + + test.fuzzy_matches.append(fuzzy_match_or_error.release_value()); + } + // Read (mis)match reference tests to load. // FIXME: Currently we only support single match or mismatch reference. String reference_to_load;