mirrors/ladybird
0
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-09-05 17:16:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Tests: Implement fuzzy screenshot comparisons in test-web

Browse source 
We now read WPT's `<meta name="fuzzy">` tags if present in tests and
apply them while comparing screenshots.
This commit is contained in:
Jelle Raaijmakers 2025-07-16 11:39:55 +02:00 • committed by Tim Ledbetter
commit 0d856a8fa7
Notes: github-actions[bot] 2025-07-17 12:00:27 +00:00
13 changed files with 260 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

20
Libraries/LibWeb/Internals/Internals.cpp
View file

@ -81,6 +81,26 @@ WebIDL::ExceptionOr<void> Internals::load_reference_test_metadata()
metadata.set("match_references"sv, TRY(collect_references("match"sv))); metadata.set("match_references"sv, TRY(collect_references("match"sv)));
metadata.set("mismatch_references"sv, TRY(collect_references("mismatch"sv))); metadata.set("mismatch_references"sv, TRY(collect_references("mismatch"sv)));
// Collect all <meta name="fuzzy" content=".."> values.
JsonArray fuzzy_configurations;
auto fuzzy_nodes = TRY(document->query_selector_all("meta[name=fuzzy]"sv));
for (size_t i = 0; i < fuzzy_nodes->length(); ++i) {
auto const* fuzzy_node = fuzzy_nodes->item(i);
auto content = as<DOM::Element>(fuzzy_node)->get_attribute_value(HTML::AttributeNames::content);
JsonObject fuzzy_configuration;
if (content.contains(':')) {
auto content_parts = MUST(content.split_limit(':', 2));
auto reference_url = document->encoding_parse_url(content_parts[0]);
fuzzy_configuration.set("reference"sv, reference_url->to_string());
content = content_parts[1];
}
fuzzy_configuration.set("content"sv, content);
fuzzy_configurations.must_append(fuzzy_configuration);
}
metadata.set("fuzzy"sv, fuzzy_configurations);
page.client().page_did_receive_reference_test_metadata(metadata); page.client().page_did_receive_reference_test_metadata(metadata);
return {}; return {};
} }

9
Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html Normal file
View file

@ -0,0 +1,9 @@
<!DOCTYPE html>
<style>
div {
width: 100px;
height: 100px;
background-color: green;
}
</style>
<div></div>

6
Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/green.html Normal file
View file

@ -0,0 +1,6 @@
<style>
:root {
print-color-adjust: exact;
background-color: green;
}
</style>

3
Tests/LibWeb/Ref/expected/wpt-import/infrastructure/reftest/red.html Normal file
View file

@ -0,0 +1,3 @@
<style>
:root {background-color:red}
</style>

12
Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_1.html Normal file
View file

@ -0,0 +1,12 @@
<!DOCTYPE html>
<link rel=match href=../../../../expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html>
<meta name=fuzzy content="../../../../expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html:255;100">
<style>
div {
width: 99px;
height: 100px;
background-color: green;
}
</style>
<div></div>

13
Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_fuzzy_no_differences.html Normal file
View file

@ -0,0 +1,13 @@
<!DOCTYPE html>
<link rel=match href=../../../../expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html>
<!-- This exactly matches the reference, and includes the possibility of
0 pixels different; in this case the maxDifference is ignored --->
<meta name=fuzzy content="../../../../expected/wpt-import/infrastructure/reftest/fuzzy-ref-1.html:128;0-100">
<style>
div {
width: 100px;
height: 100px;
background-color: green;
}
</style>
<div></div>

5
Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_match.html Normal file
View file

@ -0,0 +1,5 @@
<title>rel=match that should pass</title>
<link rel=match href=../../../../expected/wpt-import/infrastructure/reftest/green.html>
<style>
:root {background-color:green}
</style>

5
Tests/LibWeb/Ref/input/wpt-import/infrastructure/reftest/reftest_mismatch.html Normal file
View file

@ -0,0 +1,5 @@
<title>rel=mismatch that should pass</title>
<link rel=mismatch href=../../../../expected/wpt-import/infrastructure/reftest/red.html>
<style>
:root {background-color:green}
</style>

1
Tests/LibWeb/test-web/CMakeLists.txt
View file

@ -1,6 +1,7 @@
set(SOURCES set(SOURCES
Application.cpp Application.cpp
Fixture.cpp Fixture.cpp
Fuzzy.cpp
TestWebView.cpp TestWebView.cpp
main.cpp main.cpp
) )

110
Tests/LibWeb/test-web/Fuzzy.cpp Normal file
View file

@ -0,0 +1,110 @@
/*
* Copyright (c) 2025, Jelle Raaijmakers <jelle@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "Fuzzy.h"
#include <AK/Enumerate.h>
#include <AK/Format.h>
#include <LibGfx/Bitmap.h>
namespace TestWeb {
// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
bool fuzzy_screenshot_match(URL::URL const& reference, Gfx::Bitmap const& bitmap_a, Gfx::Bitmap const& bitmap_b,
Vector<FuzzyMatch> const& fuzzy_matches)
{
// If the bitmaps are identical, we don't perform fuzzy matching.
auto diff = bitmap_a.diff(bitmap_b);
if (diff.identical)
return true;
// Find a single fuzzy config to apply.
auto fuzzy_match = fuzzy_matches.first_matching([&reference](FuzzyMatch const& fuzzy_match) {
if (fuzzy_match.reference.has_value())
return fuzzy_match.reference.value().equals(reference);
return true;
});
if (!fuzzy_match.has_value())
return diff.identical;
// Apply fuzzy matching.
auto color_error_matches = fuzzy_match->color_value_error.contains(diff.maximum_error);
if (!color_error_matches)
warnln("Fuzzy mismatch: maximum error {} is outside {}", diff.maximum_error, fuzzy_match->color_value_error);
auto pixel_error_matches = fuzzy_match->pixel_error_count.contains(diff.pixel_error_count);
if (!pixel_error_matches)
warnln("Fuzzy mismatch: pixel error count {} is outside {}", diff.pixel_error_count, fuzzy_match->pixel_error_count);
return color_error_matches && pixel_error_matches;
}
// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
ErrorOr<FuzzyRange> parse_fuzzy_range(String const& fuzzy_range)
{
auto range_parts = MUST(fuzzy_range.split('-'));
if (range_parts.is_empty() || range_parts.size() > 2)
return Error::from_string_view("Invalid fuzzy range format"sv);
auto parse_value = [](String const& value) -> ErrorOr<u64> {
auto maybe_value = value.to_number<u64>();
if (!maybe_value.has_value())
return Error::from_string_view("Fuzzy range value is not a valid integer"sv);
return maybe_value.release_value();
};
auto minimum_value = TRY(parse_value(range_parts[0]));
auto maximum_value = minimum_value;
if (range_parts.size() == 2)
maximum_value = TRY(parse_value(range_parts[1]));
if (minimum_value > maximum_value)
return Error::from_string_view("Fuzzy range minimum is higher than its maximum"sv);
return FuzzyRange { .minimum_value = minimum_value, .maximum_value = maximum_value };
}
// https://web-platform-tests.org/writing-tests/reftests.html#fuzzy-matching
ErrorOr<FuzzyMatch> parse_fuzzy_match(Optional<URL::URL const&> reference, String const& content)
{
// Match configuration values. Two formats are supported:
// * Named: "maxDifference=(#X-)#Y;totalPixels=(#X-)#Y"
// * Unnamed: "(#X-)#Y;(#X-)#Y" (maxDifference and totalPixels are assumed in this order)
auto config_parts = MUST(content.split(';'));
if (config_parts.size() != 2)
return Error::from_string_view("Fuzzy configuration must have exactly two parameters"sv);
Optional<FuzzyRange> color_value_error;
Optional<FuzzyRange> pixel_error_count;
for (auto [i, config_part] : enumerate(config_parts)) {
auto named_parts = MUST(config_part.split_limit('=', 2));
if (named_parts.is_empty())
return Error::from_string_view("Fuzzy configuration value cannot be empty"sv);
if (named_parts.size() == 2) {
if (named_parts[0] == "maxDifference"sv && !color_value_error.has_value())
color_value_error = TRY(parse_fuzzy_range(named_parts[1]));
else if (named_parts[0] == "totalPixels"sv && !pixel_error_count.has_value())
pixel_error_count = TRY(parse_fuzzy_range(named_parts[1]));
else
return Error::from_string_view("Invalid fuzzy configuration parameter"sv);
} else {
if (i == 0 && !color_value_error.has_value())
color_value_error = TRY(parse_fuzzy_range(config_part));
else if (i == 1 && !pixel_error_count.has_value())
pixel_error_count = TRY(parse_fuzzy_range(config_part));
else
return Error::from_string_view("Invalid fuzzy configuration parameter"sv);
}
}
return FuzzyMatch {
.reference = reference.map([](URL::URL const& reference) { return reference; }),
.color_value_error = color_value_error.release_value(),
.pixel_error_count = pixel_error_count.release_value(),
};
}
}

47
Tests/LibWeb/test-web/Fuzzy.h Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright (c) 2025, Jelle Raaijmakers <jelle@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Error.h>
#include <AK/Optional.h>
#include <AK/String.h>
#include <AK/Types.h>
#include <LibGfx/Forward.h>
#include <LibURL/URL.h>
namespace TestWeb {
struct FuzzyRange {
u64 minimum_value;
u64 maximum_value;
bool contains(u64 value) const { return value >= minimum_value && value <= maximum_value; }
};
struct FuzzyMatch {
Optional<URL::URL> reference;
FuzzyRange color_value_error;
FuzzyRange pixel_error_count;
};
bool fuzzy_screenshot_match(URL::URL const& reference, Gfx::Bitmap const&, Gfx::Bitmap const&, Vector<FuzzyMatch> const&);
ErrorOr<FuzzyMatch> parse_fuzzy_match(Optional<URL::URL const&> reference, String const&);
ErrorOr<FuzzyRange> parse_fuzzy_range(String const&);
}
namespace AK {
template<>
struct Formatter<TestWeb::FuzzyRange> : Formatter<FormatString> {
ErrorOr<void> format(FormatBuilder& builder, TestWeb::FuzzyRange const& value)
{
return Formatter<FormatString>::format(builder, "FuzzyRange [{}-{}]"sv, value.minimum_value, value.maximum_value);
}
};
}

5
Tests/LibWeb/test-web/TestWeb.h
View file

@ -1,15 +1,19 @@
/* /*
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org> * Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
* Copyright (c) 2025, Jelle Raaijmakers <jelle@ladybird.org>
* *
* SPDX-License-Identifier: BSD-2-Clause * SPDX-License-Identifier: BSD-2-Clause
*/ */
#pragma once #pragma once
#include "Fuzzy.h"
#include <AK/ByteString.h> #include <AK/ByteString.h>
#include <AK/RefPtr.h> #include <AK/RefPtr.h>
#include <AK/String.h> #include <AK/String.h>
#include <AK/Time.h> #include <AK/Time.h>
#include <AK/Vector.h>
#include <LibCore/Promise.h> #include <LibCore/Promise.h>
#include <LibGfx/Forward.h> #include <LibGfx/Forward.h>
@ -51,6 +55,7 @@ struct Test {
bool did_finish_loading { false }; bool did_finish_loading { false };
Optional<RefTestExpectationType> ref_test_expectation_type {}; Optional<RefTestExpectationType> ref_test_expectation_type {};
Vector<FuzzyMatch> fuzzy_matches {};
RefPtr<Gfx::Bitmap const> actual_screenshot {}; RefPtr<Gfx::Bitmap const> actual_screenshot {};
RefPtr<Gfx::Bitmap const> expectation_screenshot {}; RefPtr<Gfx::Bitmap const> expectation_screenshot {};

27
Tests/LibWeb/test-web/main.cpp
View file

@ -16,7 +16,6 @@
#include <AK/Enumerate.h> #include <AK/Enumerate.h>
#include <AK/LexicalPath.h> #include <AK/LexicalPath.h>
#include <AK/QuickSort.h> #include <AK/QuickSort.h>
#include <AK/Vector.h>
#include <LibCore/ConfigFile.h> #include <LibCore/ConfigFile.h>
#include <LibCore/DirIterator.h> #include <LibCore/DirIterator.h>
#include <LibCore/Directory.h> #include <LibCore/Directory.h>
@ -375,10 +374,11 @@ static void run_ref_test(TestWebView& view, Test& test, URL::URL const& url, int
view.on_test_complete({ test, TestResult::Timeout }); view.on_test_complete({ test, TestResult::Timeout });
}); });
auto handle_completed_test = [&test, url]() -> ErrorOr<TestResult> { auto handle_completed_test = [&view, &test, url]() -> ErrorOr<TestResult> {
VERIFY(test.ref_test_expectation_type.has_value()); VERIFY(test.ref_test_expectation_type.has_value());
auto should_match = test.ref_test_expectation_type == RefTestExpectationType::Match; auto should_match = test.ref_test_expectation_type == RefTestExpectationType::Match;
auto screenshot_matches = test.actual_screenshot->diff(*test.expectation_screenshot).identical; auto screenshot_matches = fuzzy_screenshot_match(
view.url(), *test.actual_screenshot, *test.expectation_screenshot, test.fuzzy_matches);
if (should_match == screenshot_matches) if (should_match == screenshot_matches)
return TestResult::Pass; return TestResult::Pass;
@ -450,6 +450,27 @@ static void run_ref_test(TestWebView& view, Test& test, URL::URL const& url, int
auto mismatch_references = metadata_object.get_array("mismatch_references"sv); auto mismatch_references = metadata_object.get_array("mismatch_references"sv);
VERIFY(!match_references->is_empty() || !mismatch_references->is_empty()); VERIFY(!match_references->is_empty() || !mismatch_references->is_empty());
// Read fuzzy configurations.
test.fuzzy_matches.clear_with_capacity();
auto fuzzy_values = metadata_object.get_array("fuzzy"sv);
for (size_t i = 0; i < fuzzy_values->size(); ++i) {
auto fuzzy_configuration = fuzzy_values->at(i).as_object();
Optional<URL::URL> reference_url;
auto reference = fuzzy_configuration.get_string("reference"sv);
if (reference.has_value())
reference_url = URL::Parser::basic_parse(reference.release_value());
auto content = fuzzy_configuration.get_string("content"sv).release_value();
auto fuzzy_match_or_error = parse_fuzzy_match(reference_url, content);
if (fuzzy_match_or_error.is_error()) {
warnln("Failed to parse fuzzy configuration '{}' (reference: {})", content, reference_url);
continue;
}
test.fuzzy_matches.append(fuzzy_match_or_error.release_value());
}
// Read (mis)match reference tests to load. // Read (mis)match reference tests to load.
// FIXME: Currently we only support single match or mismatch reference. // FIXME: Currently we only support single match or mismatch reference.
String reference_to_load; String reference_to_load;