diff --git a/Ladybird/Headless/Application.cpp b/Ladybird/Headless/Application.cpp new file mode 100644 index 00000000000..58c4f60f723 --- /dev/null +++ b/Ladybird/Headless/Application.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +Application::Application(Badge, Main::Arguments&) + : resources_folder(s_ladybird_resource_root) + , test_concurrency(Core::System::hardware_concurrency()) +{ +} + +void Application::create_platform_arguments(Core::ArgsParser& args_parser) +{ + args_parser.add_option(screenshot_timeout, "Take a screenshot after [n] seconds (default: 1)", "screenshot", 's', "n"); + args_parser.add_option(dump_layout_tree, "Dump layout tree and exit", "dump-layout-tree", 'd'); + args_parser.add_option(dump_text, "Dump text and exit", "dump-text", 'T'); + args_parser.add_option(test_concurrency, "Maximum number of tests to run at once", "test-concurrency", 'j', "jobs"); + args_parser.add_option(test_root_path, "Run tests in path", "run-tests", 'R', "test-root-path"); + args_parser.add_option(test_glob, "Only run tests matching the given glob", "filter", 'f', "glob"); + args_parser.add_option(test_dry_run, "List the tests that would be run, without running them", "dry-run"); + args_parser.add_option(dump_failed_ref_tests, "Dump screenshots of failing ref tests", "dump-failed-ref-tests", 'D'); + args_parser.add_option(dump_gc_graph, "Dump GC graph", "dump-gc-graph", 'G'); + args_parser.add_option(resources_folder, "Path of the base resources folder (defaults to /res)", "resources", 'r', "resources-root-path"); + args_parser.add_option(is_layout_test_mode, "Enable layout test mode", "layout-test-mode"); + args_parser.add_option(rebaseline, "Rebaseline any executed layout or text tests", "rebaseline"); +} + +void Application::create_platform_options(WebView::ChromeOptions& chrome_options, WebView::WebContentOptions& web_content_options) +{ + if (!test_root_path.is_empty()) { + // --run-tests implies --layout-test-mode. + is_layout_test_mode = true; + } + + if (is_layout_test_mode) { + // Allow window.open() to succeed for tests. + chrome_options.allow_popups = WebView::AllowPopups::Yes; + } + + if (dump_gc_graph) { + // Force all tests to run in serial if we are interested in the GC graph. + test_concurrency = 1; + } + + web_content_options.is_layout_test_mode = is_layout_test_mode ? WebView::IsLayoutTestMode::Yes : WebView::IsLayoutTestMode::No; +} + +ErrorOr Application::launch_services() +{ + auto request_server_paths = TRY(get_paths_for_helper_process("RequestServer"sv)); + m_request_client = TRY(launch_request_server_process(request_server_paths, resources_folder)); + + auto image_decoder_paths = TRY(get_paths_for_helper_process("ImageDecoder"sv)); + m_image_decoder_client = TRY(launch_image_decoder_process(image_decoder_paths)); + + return {}; +} + +ErrorOr Application::create_web_view(Core::AnonymousBuffer theme, Gfx::IntSize window_size) +{ + auto web_view = TRY(HeadlessWebView::create(move(theme), window_size)); + m_web_views.append(move(web_view)); + + return m_web_views.last().ptr(); +} + +void Application::destroy_web_views() +{ + m_web_views.clear(); +} + +} diff --git a/Ladybird/Headless/Application.h b/Ladybird/Headless/Application.h new file mode 100644 index 00000000000..8b31185ec56 --- /dev/null +++ b/Ladybird/Headless/Application.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +class HeadlessWebView; + +class Application : public WebView::Application { + WEB_VIEW_APPLICATION(Application) + +public: + static Application& the() + { + return static_cast(WebView::Application::the()); + } + + virtual void create_platform_arguments(Core::ArgsParser&) override; + virtual void create_platform_options(WebView::ChromeOptions&, WebView::WebContentOptions&) override; + + ErrorOr launch_services(); + + static Requests::RequestClient& request_client() { return *the().m_request_client; } + static ImageDecoderClient::Client& image_decoder_client() { return *the().m_image_decoder_client; } + + ErrorOr create_web_view(Core::AnonymousBuffer theme, Gfx::IntSize window_size); + void destroy_web_views(); + + template + void for_each_web_view(Callback&& callback) + { + for (auto& web_view : m_web_views) + callback(*web_view); + } + + int screenshot_timeout { 1 }; + ByteString resources_folder; + bool dump_failed_ref_tests { false }; + bool dump_layout_tree { false }; + bool dump_text { false }; + bool dump_gc_graph { false }; + bool is_layout_test_mode { false }; + size_t test_concurrency { 1 }; + ByteString test_root_path; + ByteString test_glob; + bool test_dry_run { false }; + bool rebaseline { false }; + +private: + RefPtr m_request_client; + RefPtr m_image_decoder_client; + + Vector> m_web_views; +}; + +} diff --git a/Ladybird/Headless/CMakeLists.txt b/Ladybird/Headless/CMakeLists.txt index 337d17fec5b..14b28d233ec 100644 --- a/Ladybird/Headless/CMakeLists.txt +++ b/Ladybird/Headless/CMakeLists.txt @@ -1,5 +1,8 @@ set(SOURCES ${LADYBIRD_SOURCES} + Application.cpp + HeadlessWebView.cpp + Test.cpp main.cpp ) diff --git a/Ladybird/Headless/HeadlessWebView.cpp b/Ladybird/Headless/HeadlessWebView.cpp new file mode 100644 index 00000000000..9b057df3190 --- /dev/null +++ b/Ladybird/Headless/HeadlessWebView.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +HeadlessWebView::HeadlessWebView(Gfx::IntSize viewport_size) + : m_viewport_size(viewport_size) + , m_test_promise(TestPromise::construct()) +{ + on_request_worker_agent = []() { + auto web_worker_paths = MUST(get_paths_for_helper_process("WebWorker"sv)); + auto worker_client = MUST(launch_web_worker_process(web_worker_paths, Application::request_client())); + + return worker_client->dup_socket(); + }; +} + +ErrorOr> HeadlessWebView::create(Core::AnonymousBuffer theme, Gfx::IntSize window_size) +{ + auto view = TRY(adopt_nonnull_own_or_enomem(new (nothrow) HeadlessWebView(window_size))); + + auto request_server_socket = TRY(connect_new_request_server_client(Application::request_client())); + auto image_decoder_socket = TRY(connect_new_image_decoder_client(Application::image_decoder_client())); + + auto candidate_web_content_paths = TRY(get_paths_for_helper_process("WebContent"sv)); + view->m_client_state.client = TRY(launch_web_content_process(*view, candidate_web_content_paths, move(image_decoder_socket), move(request_server_socket))); + + view->client().async_update_system_theme(0, move(theme)); + view->client().async_set_viewport_size(0, view->viewport_size()); + view->client().async_set_window_size(0, view->viewport_size()); + + if (WebView::Application::chrome_options().allow_popups == WebView::AllowPopups::Yes) + view->client().async_debug_request(0, "block-pop-ups"sv, "off"sv); + + if (auto web_driver_ipc_path = WebView::Application::chrome_options().webdriver_content_ipc_path; web_driver_ipc_path.has_value()) + view->client().async_connect_to_webdriver(0, *web_driver_ipc_path); + + view->m_client_state.client->on_web_content_process_crash = [&view = *view] { + warnln("\033[31;1mWebContent Crashed!!\033[0m"); + warnln(" Last page loaded: {}", view.url()); + VERIFY_NOT_REACHED(); + }; + + return view; +} + +void HeadlessWebView::clear_content_filters() +{ + client().async_set_content_filters(0, {}); +} + +NonnullRefPtr>> HeadlessWebView::take_screenshot() +{ + VERIFY(!m_pending_screenshot); + + m_pending_screenshot = Core::Promise>::construct(); + client().async_take_document_screenshot(0); + + return *m_pending_screenshot; +} + +void HeadlessWebView::did_receive_screenshot(Badge, Gfx::ShareableBitmap const& screenshot) +{ + VERIFY(m_pending_screenshot); + + auto pending_screenshot = move(m_pending_screenshot); + pending_screenshot->resolve(screenshot.bitmap()); +} + +void HeadlessWebView::on_test_complete(TestCompletion completion) +{ + m_test_promise->resolve(move(completion)); +} + +} diff --git a/Ladybird/Headless/HeadlessWebView.h b/Ladybird/Headless/HeadlessWebView.h new file mode 100644 index 00000000000..1f1c806471c --- /dev/null +++ b/Ladybird/Headless/HeadlessWebView.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +class HeadlessWebView final : public WebView::ViewImplementation { +public: + static ErrorOr> create(Core::AnonymousBuffer theme, Gfx::IntSize window_size); + + void clear_content_filters(); + + NonnullRefPtr>> take_screenshot(); + + TestPromise& test_promise() { return *m_test_promise; } + void on_test_complete(TestCompletion); + +private: + explicit HeadlessWebView(Gfx::IntSize viewport_size); + + void update_zoom() override { } + void initialize_client(CreateNewClient) override { } + + virtual Web::DevicePixelSize viewport_size() const override { return m_viewport_size.to_type(); } + virtual Gfx::IntPoint to_content_position(Gfx::IntPoint widget_position) const override { return widget_position; } + virtual Gfx::IntPoint to_widget_position(Gfx::IntPoint content_position) const override { return content_position; } + + virtual void did_receive_screenshot(Badge, Gfx::ShareableBitmap const& screenshot) override; + + Gfx::IntSize m_viewport_size; + RefPtr>> m_pending_screenshot; + + NonnullRefPtr m_test_promise; +}; + +} diff --git a/Ladybird/Headless/Test.cpp b/Ladybird/Headless/Test.cpp new file mode 100644 index 00000000000..9fd1ee24137 --- /dev/null +++ b/Ladybird/Headless/Test.cpp @@ -0,0 +1,510 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +static Vector s_skipped_tests; + +static ErrorOr load_test_config(StringView test_root_path) +{ + auto config_path = LexicalPath::join(test_root_path, "TestConfig.ini"sv); + auto config_or_error = Core::ConfigFile::open(config_path.string()); + + if (config_or_error.is_error()) { + if (config_or_error.error().code() == ENOENT) + return {}; + warnln("Unable to open test config {}", config_path); + return config_or_error.release_error(); + } + + auto config = config_or_error.release_value(); + + for (auto const& group : config->groups()) { + if (group == "Skipped"sv) { + for (auto& key : config->keys(group)) + s_skipped_tests.append(LexicalPath::join(test_root_path, key).string()); + } else { + warnln("Unknown group '{}' in config {}", group, config_path); + } + } + + return {}; +} + +static ErrorOr collect_dump_tests(Vector& tests, StringView path, StringView trail, TestMode mode) +{ + Core::DirIterator it(ByteString::formatted("{}/input/{}", path, trail), Core::DirIterator::Flags::SkipDots); + + while (it.has_next()) { + auto name = it.next_path(); + auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/input/{}/{}", path, trail, name))); + + if (FileSystem::is_directory(input_path)) { + TRY(collect_dump_tests(tests, path, ByteString::formatted("{}/{}", trail, name), mode)); + continue; + } + + if (!name.ends_with(".html"sv) && !name.ends_with(".svg"sv)) + continue; + + auto expectation_path = ByteString::formatted("{}/expected/{}/{}.txt", path, trail, LexicalPath::title(name)); + tests.append({ mode, input_path, move(expectation_path), {} }); + } + + return {}; +} + +static ErrorOr collect_ref_tests(Vector& tests, StringView path) +{ + TRY(Core::Directory::for_each_entry(path, Core::DirIterator::SkipDots, [&](Core::DirectoryEntry const& entry, Core::Directory const&) -> ErrorOr { + if (entry.type == Core::DirectoryEntry::Type::Directory) + return IterationDecision::Continue; + + auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/{}", path, entry.name))); + tests.append({ TestMode::Ref, input_path, {}, {} }); + + return IterationDecision::Continue; + })); + + return {}; +} + +void run_dump_test(HeadlessWebView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds) +{ + auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() { + view.on_load_finish = {}; + view.on_text_test_finish = {}; + + view.on_test_complete({ test, TestResult::Timeout }); + }); + + auto handle_completed_test = [&test, url]() -> ErrorOr { + if (test.expectation_path.is_empty()) { + outln("{}", test.text); + return TestResult::Pass; + } + + auto open_expectation_file = [&](auto mode) { + auto expectation_file_or_error = Core::File::open(test.expectation_path, mode); + if (expectation_file_or_error.is_error()) + warnln("Failed opening '{}': {}", test.expectation_path, expectation_file_or_error.error()); + + return expectation_file_or_error; + }; + + ByteBuffer expectation; + + if (auto expectation_file = open_expectation_file(Core::File::OpenMode::Read); !expectation_file.is_error()) { + expectation = TRY(expectation_file.value()->read_until_eof()); + + auto result_trimmed = StringView { test.text }.trim("\n"sv, TrimMode::Right); + auto expectation_trimmed = StringView { expectation }.trim("\n"sv, TrimMode::Right); + + if (result_trimmed == expectation_trimmed) + return TestResult::Pass; + } else if (!Application::the().rebaseline) { + return expectation_file.release_error(); + } + + if (Application::the().rebaseline) { + auto expectation_file = TRY(open_expectation_file(Core::File::OpenMode::Write)); + TRY(expectation_file->write_until_depleted(test.text)); + + return TestResult::Pass; + } + + auto const color_output = isatty(STDOUT_FILENO) ? Diff::ColorOutput::Yes : Diff::ColorOutput::No; + + if (color_output == Diff::ColorOutput::Yes) + outln("\n\033[33;1mTest failed\033[0m: {}", url); + else + outln("\nTest failed: {}", url); + + auto hunks = TRY(Diff::from_text(expectation, test.text, 3)); + auto out = TRY(Core::File::standard_output()); + + TRY(Diff::write_unified_header(test.expectation_path, test.expectation_path, *out)); + for (auto const& hunk : hunks) + TRY(Diff::write_unified(hunk, *out, color_output)); + + return TestResult::Fail; + }; + + auto on_test_complete = [&view, &test, timer, handle_completed_test]() { + timer->stop(); + + view.on_load_finish = {}; + view.on_text_test_finish = {}; + + if (auto result = handle_completed_test(); result.is_error()) + view.on_test_complete({ test, TestResult::Fail }); + else + view.on_test_complete({ test, result.value() }); + }; + + if (test.mode == TestMode::Layout) { + view.on_load_finish = [&view, &test, url, on_test_complete = move(on_test_complete)](auto const& loaded_url) { + // We don't want subframe loads to trigger the test finish. + if (!url.equals(loaded_url, URL::ExcludeFragment::Yes)) + return; + + // NOTE: We take a screenshot here to force the lazy layout of SVG-as-image documents to happen. + // It also causes a lot more code to run, which is good for finding bugs. :^) + view.take_screenshot()->when_resolved([&view, &test, on_test_complete = move(on_test_complete)](auto) { + auto promise = view.request_internal_page_info(WebView::PageInfoType::LayoutTree | WebView::PageInfoType::PaintTree); + + promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) { + test.text = text; + on_test_complete(); + }); + }); + }; + } else if (test.mode == TestMode::Text) { + view.on_load_finish = [&view, &test, on_test_complete, url](auto const& loaded_url) { + // We don't want subframe loads to trigger the test finish. + if (!url.equals(loaded_url, URL::ExcludeFragment::Yes)) + return; + + test.did_finish_loading = true; + + if (test.expectation_path.is_empty()) { + auto promise = view.request_internal_page_info(WebView::PageInfoType::Text); + + promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) { + test.text = text; + on_test_complete(); + }); + } else if (test.did_finish_test) { + on_test_complete(); + } + }; + + view.on_text_test_finish = [&test, on_test_complete](auto const& text) { + test.text = text; + test.did_finish_test = true; + + if (test.did_finish_loading) + on_test_complete(); + }; + } + + view.load(url); + timer->start(); +} + +static void run_ref_test(HeadlessWebView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds = DEFAULT_TIMEOUT_MS) +{ + auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() { + view.on_load_finish = {}; + view.on_text_test_finish = {}; + + view.on_test_complete({ test, TestResult::Timeout }); + }); + + auto handle_completed_test = [&test, url]() -> ErrorOr { + if (test.actual_screenshot->visually_equals(*test.expectation_screenshot)) + return TestResult::Pass; + + if (Application::the().dump_failed_ref_tests) { + warnln("\033[33;1mRef test {} failed; dumping screenshots\033[0m", url); + + auto dump_screenshot = [&](Gfx::Bitmap& bitmap, StringView path) -> ErrorOr { + auto screenshot_file = TRY(Core::File::open(path, Core::File::OpenMode::Write)); + auto encoded_data = TRY(Gfx::PNGWriter::encode(bitmap)); + TRY(screenshot_file->write_until_depleted(encoded_data)); + + outln("\033[33;1mDumped {}\033[0m", TRY(FileSystem::real_path(path))); + return {}; + }; + + TRY(Core::Directory::create("test-dumps"sv, Core::Directory::CreateDirectories::Yes)); + + auto title = LexicalPath::title(URL::percent_decode(url.serialize_path())); + TRY(dump_screenshot(*test.actual_screenshot, ByteString::formatted("test-dumps/{}.png", title))); + TRY(dump_screenshot(*test.expectation_screenshot, ByteString::formatted("test-dumps/{}-ref.png", title))); + } + + return TestResult::Fail; + }; + + auto on_test_complete = [&view, &test, timer, handle_completed_test]() { + timer->stop(); + + view.on_load_finish = {}; + view.on_text_test_finish = {}; + + if (auto result = handle_completed_test(); result.is_error()) + view.on_test_complete({ test, TestResult::Fail }); + else + view.on_test_complete({ test, result.value() }); + }; + + view.on_load_finish = [&view, &test, on_test_complete = move(on_test_complete)](auto const&) { + if (test.actual_screenshot) { + view.take_screenshot()->when_resolved([&test, on_test_complete = move(on_test_complete)](RefPtr screenshot) { + test.expectation_screenshot = move(screenshot); + on_test_complete(); + }); + } else { + view.take_screenshot()->when_resolved([&view, &test](RefPtr screenshot) { + test.actual_screenshot = move(screenshot); + view.debug_request("load-reference-page"); + }); + } + }; + + view.on_text_test_finish = [&](auto const&) { + dbgln("Unexpected text test finished during ref test for {}", url); + }; + + view.load(url); + timer->start(); +} + +static void run_test(HeadlessWebView& view, Test& test) +{ + // Clear the current document. + // FIXME: Implement a debug-request to do this more thoroughly. + auto promise = Core::Promise::construct(); + + view.on_load_finish = [promise](auto const& url) { + if (!url.equals("about:blank"sv)) + return; + + Core::deferred_invoke([promise]() { + promise->resolve({}); + }); + }; + + view.on_text_test_finish = {}; + + view.on_request_file_picker = [&](auto const& accepted_file_types, auto allow_multiple_files) { + // Create some dummy files for tests. + Vector selected_files; + + bool add_txt_files = accepted_file_types.filters.is_empty(); + bool add_cpp_files = false; + + for (auto const& filter : accepted_file_types.filters) { + filter.visit( + [](Web::HTML::FileFilter::FileType) {}, + [&](Web::HTML::FileFilter::MimeType const& mime_type) { + if (mime_type.value == "text/plain"sv) + add_txt_files = true; + }, + [&](Web::HTML::FileFilter::Extension const& extension) { + if (extension.value == "cpp"sv) + add_cpp_files = true; + }); + } + + if (add_txt_files) { + selected_files.empend("file1"sv, MUST(ByteBuffer::copy("Contents for file1"sv.bytes()))); + + if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) { + selected_files.empend("file2"sv, MUST(ByteBuffer::copy("Contents for file2"sv.bytes()))); + selected_files.empend("file3"sv, MUST(ByteBuffer::copy("Contents for file3"sv.bytes()))); + selected_files.empend("file4"sv, MUST(ByteBuffer::copy("Contents for file4"sv.bytes()))); + } + } + + if (add_cpp_files) { + selected_files.empend("file1.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 1; }}"sv.bytes()))); + + if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) { + selected_files.empend("file2.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 2; }}"sv.bytes()))); + } + } + + view.file_picker_closed(move(selected_files)); + }; + + promise->when_resolved([&view, &test](auto) { + auto url = URL::create_with_file_scheme(MUST(FileSystem::real_path(test.input_path))); + + switch (test.mode) { + case TestMode::Text: + case TestMode::Layout: + run_dump_test(view, test, url); + return; + case TestMode::Ref: + run_ref_test(view, test, url); + return; + } + + VERIFY_NOT_REACHED(); + }); + + view.load("about:blank"sv); +} + +ErrorOr run_tests(Core::AnonymousBuffer const& theme, Gfx::IntSize window_size) +{ + auto& app = Application::the(); + TRY(load_test_config(app.test_root_path)); + + Vector tests; + auto test_glob = ByteString::formatted("*{}*", app.test_glob); + + TRY(collect_dump_tests(tests, ByteString::formatted("{}/Layout", app.test_root_path), "."sv, TestMode::Layout)); + TRY(collect_dump_tests(tests, ByteString::formatted("{}/Text", app.test_root_path), "."sv, TestMode::Text)); + TRY(collect_ref_tests(tests, ByteString::formatted("{}/Ref", app.test_root_path))); +#if !defined(AK_OS_MACOS) + TRY(collect_ref_tests(tests, ByteString::formatted("{}/Screenshot", app.test_root_path))); +#endif + + tests.remove_all_matching([&](auto const& test) { + return !test.input_path.matches(test_glob, CaseSensitivity::CaseSensitive); + }); + + if (app.test_dry_run) { + outln("Found {} tests...", tests.size()); + + for (auto const& [i, test] : enumerate(tests)) + outln("{}/{}: {}", i + 1, tests.size(), LexicalPath::relative_path(test.input_path, app.test_root_path)); + + return {}; + } + + auto concurrency = min(app.test_concurrency, tests.size()); + size_t loaded_web_views = 0; + + for (size_t i = 0; i < concurrency; ++i) { + auto& view = *TRY(app.create_web_view(theme, window_size)); + view.on_load_finish = [&](auto const&) { ++loaded_web_views; }; + } + + // We need to wait for the initial about:blank load to complete before starting the tests, otherwise we may load the + // test URL before the about:blank load completes. WebContent currently cannot handle this, and will drop the test URL. + Core::EventLoop::current().spin_until([&]() { + return loaded_web_views == concurrency; + }); + + size_t pass_count = 0; + size_t fail_count = 0; + size_t timeout_count = 0; + size_t skipped_count = 0; + + bool is_tty = isatty(STDOUT_FILENO); + outln("Running {} tests...", tests.size()); + + auto all_tests_complete = Core::Promise::construct(); + auto tests_remaining = tests.size(); + auto current_test = 0uz; + + Vector non_passing_tests; + + app.for_each_web_view([&](auto& view) { + view.clear_content_filters(); + + auto run_next_test = [&]() { + auto index = current_test++; + if (index >= tests.size()) + return; + auto& test = tests[index]; + + if (is_tty) { + // Keep clearing and reusing the same line if stdout is a TTY. + out("\33[2K\r"); + } + + out("{}/{}: {}", index + 1, tests.size(), LexicalPath::relative_path(test.input_path, app.test_root_path)); + + if (is_tty) + fflush(stdout); + else + outln(""); + + Core::deferred_invoke([&]() mutable { + if (s_skipped_tests.contains_slow(test.input_path)) + view.on_test_complete({ test, TestResult::Skipped }); + else + run_test(view, test); + }); + }; + + view.test_promise().when_resolved([&, run_next_test](auto result) { + switch (result.result) { + case TestResult::Pass: + ++pass_count; + break; + case TestResult::Fail: + ++fail_count; + break; + case TestResult::Timeout: + ++timeout_count; + break; + case TestResult::Skipped: + ++skipped_count; + break; + } + + if (result.result != TestResult::Pass) + non_passing_tests.append(move(result)); + + if (--tests_remaining == 0) + all_tests_complete->resolve({}); + else + run_next_test(); + }); + + Core::deferred_invoke([run_next_test]() { + run_next_test(); + }); + }); + + MUST(all_tests_complete->await()); + + if (is_tty) + outln("\33[2K\rDone!"); + + outln("=================================================="); + outln("Pass: {}, Fail: {}, Skipped: {}, Timeout: {}", pass_count, fail_count, skipped_count, timeout_count); + outln("=================================================="); + + for (auto const& non_passing_test : non_passing_tests) + outln("{}: {}", test_result_to_string(non_passing_test.result), non_passing_test.test.input_path); + + if (app.dump_gc_graph) { + app.for_each_web_view([&](auto& view) { + if (auto path = view.dump_gc_graph(); path.is_error()) + warnln("Failed to dump GC graph: {}", path.error()); + else + outln("GC graph dumped to {}", path.value()); + }); + } + + app.destroy_web_views(); + + if (timeout_count == 0 && fail_count == 0) + return {}; + + return Error::from_string_literal("Failed LibWeb tests"); +} + +} diff --git a/Ladybird/Headless/Test.h b/Ladybird/Headless/Test.h new file mode 100644 index 00000000000..709483515dd --- /dev/null +++ b/Ladybird/Headless/Test.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +class HeadlessWebView; + +enum class TestMode { + Layout, + Text, + Ref, +}; + +enum class TestResult { + Pass, + Fail, + Skipped, + Timeout, +}; + +static constexpr StringView test_result_to_string(TestResult result) +{ + switch (result) { + case TestResult::Pass: + return "Pass"sv; + case TestResult::Fail: + return "Fail"sv; + case TestResult::Skipped: + return "Skipped"sv; + case TestResult::Timeout: + return "Timeout"sv; + } + VERIFY_NOT_REACHED(); +} + +struct Test { + TestMode mode; + + ByteString input_path {}; + ByteString expectation_path {}; + + String text {}; + bool did_finish_test { false }; + bool did_finish_loading { false }; + + RefPtr actual_screenshot {}; + RefPtr expectation_screenshot {}; +}; + +struct TestCompletion { + Test& test; + TestResult result; +}; + +using TestPromise = Core::Promise; + +constexpr inline int DEFAULT_TIMEOUT_MS = 30000; // 30sec + +ErrorOr run_tests(Core::AnonymousBuffer const& theme, Gfx::IntSize window_size); +void run_dump_test(HeadlessWebView&, Test&, URL::URL const&, int timeout_in_milliseconds = DEFAULT_TIMEOUT_MS); + +} diff --git a/Ladybird/Headless/main.cpp b/Ladybird/Headless/main.cpp index fef52063a69..660158d5136 100644 --- a/Ladybird/Headless/main.cpp +++ b/Ladybird/Headless/main.cpp @@ -7,291 +7,26 @@ * SPDX-License-Identifier: BSD-2-Clause */ -#include -#include #include -#include #include -#include #include #include -#include -#include +#include +#include +#include #include -#include -#include -#include -#include #include #include #include #include -#include #include -#include -#include #include #include -#include #include -#include -#include -#include #include -#include -#include -#include #include -#include -#include -#include -#include -constexpr int DEFAULT_TIMEOUT_MS = 30000; // 30sec - -enum class TestMode { - Layout, - Text, - Ref, -}; - -enum class TestResult { - Pass, - Fail, - Skipped, - Timeout, -}; - -static constexpr StringView test_result_to_string(TestResult result) -{ - switch (result) { - case TestResult::Pass: - return "Pass"sv; - case TestResult::Fail: - return "Fail"sv; - case TestResult::Skipped: - return "Skipped"sv; - case TestResult::Timeout: - return "Timeout"sv; - } - VERIFY_NOT_REACHED(); -} - -struct Test { - TestMode mode; - - ByteString input_path {}; - ByteString expectation_path {}; - - String text {}; - bool did_finish_test { false }; - bool did_finish_loading { false }; - - RefPtr actual_screenshot {}; - RefPtr expectation_screenshot {}; -}; - -struct TestCompletion { - Test& test; - TestResult result; -}; - -using TestPromise = Core::Promise; - -class HeadlessWebContentView; - -class Application : public WebView::Application { - WEB_VIEW_APPLICATION(Application) - -public: - static Application& the() - { - return static_cast(WebView::Application::the()); - } - - virtual void create_platform_arguments(Core::ArgsParser& args_parser) override - { - args_parser.add_option(screenshot_timeout, "Take a screenshot after [n] seconds (default: 1)", "screenshot", 's', "n"); - args_parser.add_option(dump_layout_tree, "Dump layout tree and exit", "dump-layout-tree", 'd'); - args_parser.add_option(dump_text, "Dump text and exit", "dump-text", 'T'); - args_parser.add_option(test_concurrency, "Maximum number of tests to run at once", "test-concurrency", 'j', "jobs"); - args_parser.add_option(test_root_path, "Run tests in path", "run-tests", 'R', "test-root-path"); - args_parser.add_option(test_glob, "Only run tests matching the given glob", "filter", 'f', "glob"); - args_parser.add_option(test_dry_run, "List the tests that would be run, without running them", "dry-run"); - args_parser.add_option(dump_failed_ref_tests, "Dump screenshots of failing ref tests", "dump-failed-ref-tests", 'D'); - args_parser.add_option(dump_gc_graph, "Dump GC graph", "dump-gc-graph", 'G'); - args_parser.add_option(resources_folder, "Path of the base resources folder (defaults to /res)", "resources", 'r', "resources-root-path"); - args_parser.add_option(is_layout_test_mode, "Enable layout test mode", "layout-test-mode"); - args_parser.add_option(rebaseline, "Rebaseline any executed layout or text tests", "rebaseline"); - } - - virtual void create_platform_options(WebView::ChromeOptions& chrome_options, WebView::WebContentOptions& web_content_options) override - { - if (!test_root_path.is_empty()) { - // --run-tests implies --layout-test-mode. - is_layout_test_mode = true; - } - - if (is_layout_test_mode) { - // Allow window.open() to succeed for tests. - chrome_options.allow_popups = WebView::AllowPopups::Yes; - } - - if (dump_gc_graph) { - // Force all tests to run in serial if we are interested in the GC graph. - test_concurrency = 1; - } - - web_content_options.is_layout_test_mode = is_layout_test_mode ? WebView::IsLayoutTestMode::Yes : WebView::IsLayoutTestMode::No; - } - - ErrorOr launch_services() - { - auto request_server_paths = TRY(get_paths_for_helper_process("RequestServer"sv)); - m_request_client = TRY(launch_request_server_process(request_server_paths, resources_folder)); - - auto image_decoder_paths = TRY(get_paths_for_helper_process("ImageDecoder"sv)); - m_image_decoder_client = TRY(launch_image_decoder_process(image_decoder_paths)); - - return {}; - } - - static Requests::RequestClient& request_client() { return *the().m_request_client; } - static ImageDecoderClient::Client& image_decoder_client() { return *the().m_image_decoder_client; } - - ErrorOr create_web_view(Core::AnonymousBuffer theme, Gfx::IntSize window_size); - void destroy_web_views(); - - template - void for_each_web_view(Callback&& callback) - { - for (auto& web_view : m_web_views) - callback(*web_view); - } - - int screenshot_timeout { 1 }; - ByteString resources_folder { s_ladybird_resource_root }; - bool dump_failed_ref_tests { false }; - bool dump_layout_tree { false }; - bool dump_text { false }; - bool dump_gc_graph { false }; - bool is_layout_test_mode { false }; - size_t test_concurrency { Core::System::hardware_concurrency() }; - ByteString test_root_path; - ByteString test_glob; - bool test_dry_run { false }; - bool rebaseline { false }; - -private: - RefPtr m_request_client; - RefPtr m_image_decoder_client; - - Vector> m_web_views; -}; - -Application::Application(Badge, Main::Arguments&) -{ -} - -class HeadlessWebContentView final : public WebView::ViewImplementation { -public: - static ErrorOr> create(Core::AnonymousBuffer theme, Gfx::IntSize window_size) - { - auto view = TRY(adopt_nonnull_own_or_enomem(new (nothrow) HeadlessWebContentView(window_size))); - - auto request_server_socket = TRY(connect_new_request_server_client(Application::request_client())); - auto image_decoder_socket = TRY(connect_new_image_decoder_client(Application::image_decoder_client())); - - auto candidate_web_content_paths = TRY(get_paths_for_helper_process("WebContent"sv)); - view->m_client_state.client = TRY(launch_web_content_process(*view, candidate_web_content_paths, move(image_decoder_socket), move(request_server_socket))); - - view->client().async_update_system_theme(0, move(theme)); - view->client().async_set_viewport_size(0, view->viewport_size()); - view->client().async_set_window_size(0, view->viewport_size()); - - if (WebView::Application::chrome_options().allow_popups == WebView::AllowPopups::Yes) - view->client().async_debug_request(0, "block-pop-ups"sv, "off"sv); - - if (auto web_driver_ipc_path = WebView::Application::chrome_options().webdriver_content_ipc_path; web_driver_ipc_path.has_value()) - view->client().async_connect_to_webdriver(0, *web_driver_ipc_path); - - view->m_client_state.client->on_web_content_process_crash = [&view = *view] { - warnln("\033[31;1mWebContent Crashed!!\033[0m"); - warnln(" Last page loaded: {}", view.url()); - VERIFY_NOT_REACHED(); - }; - - return view; - } - - NonnullRefPtr>> take_screenshot() - { - VERIFY(!m_pending_screenshot); - - m_pending_screenshot = Core::Promise>::construct(); - client().async_take_document_screenshot(0); - - return *m_pending_screenshot; - } - - void clear_content_filters() - { - client().async_set_content_filters(0, {}); - } - - TestPromise& test_promise() { return *m_test_promise; } - - void on_test_complete(TestCompletion completion) - { - m_test_promise->resolve(move(completion)); - } - -private: - HeadlessWebContentView(Gfx::IntSize viewport_size) - : m_viewport_size(viewport_size) - , m_test_promise(TestPromise::construct()) - { - on_request_worker_agent = []() { - auto worker_client = MUST(launch_web_worker_process(MUST(get_paths_for_helper_process("WebWorker"sv)), Application::request_client())); - return worker_client->dup_socket(); - }; - } - - void update_zoom() override { } - void initialize_client(CreateNewClient) override { } - - virtual void did_receive_screenshot(Badge, Gfx::ShareableBitmap const& screenshot) override - { - VERIFY(m_pending_screenshot); - - auto pending_screenshot = move(m_pending_screenshot); - pending_screenshot->resolve(screenshot.bitmap()); - } - - virtual Web::DevicePixelSize viewport_size() const override { return m_viewport_size.to_type(); } - virtual Gfx::IntPoint to_content_position(Gfx::IntPoint widget_position) const override { return widget_position; } - virtual Gfx::IntPoint to_widget_position(Gfx::IntPoint content_position) const override { return content_position; } - - Gfx::IntSize m_viewport_size; - RefPtr>> m_pending_screenshot; - - NonnullRefPtr m_test_promise; -}; - -ErrorOr Application::create_web_view(Core::AnonymousBuffer theme, Gfx::IntSize window_size) -{ - auto web_view = TRY(HeadlessWebContentView::create(move(theme), window_size)); - m_web_views.append(move(web_view)); - - return m_web_views.last().ptr(); -} - -void Application::destroy_web_views() -{ - m_web_views.clear(); -} - -static ErrorOr> load_page_for_screenshot_and_exit(Core::EventLoop& event_loop, HeadlessWebContentView& view, URL::URL const& url, int screenshot_timeout) +static ErrorOr> load_page_for_screenshot_and_exit(Core::EventLoop& event_loop, Ladybird::HeadlessWebView& view, URL::URL const& url, int screenshot_timeout) { // FIXME: Allow passing the output path as an argument. static constexpr auto output_file_path = "output.png"sv; @@ -324,483 +59,11 @@ static ErrorOr> load_page_for_screenshot_and_exit(Cor return timer; } -static void run_dump_test(HeadlessWebContentView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds = DEFAULT_TIMEOUT_MS) -{ - auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() { - view.on_load_finish = {}; - view.on_text_test_finish = {}; - - view.on_test_complete({ test, TestResult::Timeout }); - }); - - auto handle_completed_test = [&test, url]() -> ErrorOr { - if (test.expectation_path.is_empty()) { - outln("{}", test.text); - return TestResult::Pass; - } - - auto open_expectation_file = [&](auto mode) { - auto expectation_file_or_error = Core::File::open(test.expectation_path, mode); - if (expectation_file_or_error.is_error()) - warnln("Failed opening '{}': {}", test.expectation_path, expectation_file_or_error.error()); - - return expectation_file_or_error; - }; - - ByteBuffer expectation; - - if (auto expectation_file = open_expectation_file(Core::File::OpenMode::Read); !expectation_file.is_error()) { - expectation = TRY(expectation_file.value()->read_until_eof()); - - auto result_trimmed = StringView { test.text }.trim("\n"sv, TrimMode::Right); - auto expectation_trimmed = StringView { expectation }.trim("\n"sv, TrimMode::Right); - - if (result_trimmed == expectation_trimmed) - return TestResult::Pass; - } else if (!Application::the().rebaseline) { - return expectation_file.release_error(); - } - - if (Application::the().rebaseline) { - auto expectation_file = TRY(open_expectation_file(Core::File::OpenMode::Write)); - TRY(expectation_file->write_until_depleted(test.text)); - - return TestResult::Pass; - } - - auto const color_output = isatty(STDOUT_FILENO) ? Diff::ColorOutput::Yes : Diff::ColorOutput::No; - - if (color_output == Diff::ColorOutput::Yes) - outln("\n\033[33;1mTest failed\033[0m: {}", url); - else - outln("\nTest failed: {}", url); - - auto hunks = TRY(Diff::from_text(expectation, test.text, 3)); - auto out = TRY(Core::File::standard_output()); - - TRY(Diff::write_unified_header(test.expectation_path, test.expectation_path, *out)); - for (auto const& hunk : hunks) - TRY(Diff::write_unified(hunk, *out, color_output)); - - return TestResult::Fail; - }; - - auto on_test_complete = [&view, &test, timer, handle_completed_test]() { - timer->stop(); - - view.on_load_finish = {}; - view.on_text_test_finish = {}; - - if (auto result = handle_completed_test(); result.is_error()) - view.on_test_complete({ test, TestResult::Fail }); - else - view.on_test_complete({ test, result.value() }); - }; - - if (test.mode == TestMode::Layout) { - view.on_load_finish = [&view, &test, url, on_test_complete = move(on_test_complete)](auto const& loaded_url) { - // We don't want subframe loads to trigger the test finish. - if (!url.equals(loaded_url, URL::ExcludeFragment::Yes)) - return; - - // NOTE: We take a screenshot here to force the lazy layout of SVG-as-image documents to happen. - // It also causes a lot more code to run, which is good for finding bugs. :^) - view.take_screenshot()->when_resolved([&view, &test, on_test_complete = move(on_test_complete)](auto) { - auto promise = view.request_internal_page_info(WebView::PageInfoType::LayoutTree | WebView::PageInfoType::PaintTree); - - promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) { - test.text = text; - on_test_complete(); - }); - }); - }; - } else if (test.mode == TestMode::Text) { - view.on_load_finish = [&view, &test, on_test_complete, url](auto const& loaded_url) { - // We don't want subframe loads to trigger the test finish. - if (!url.equals(loaded_url, URL::ExcludeFragment::Yes)) - return; - - test.did_finish_loading = true; - - if (test.expectation_path.is_empty()) { - auto promise = view.request_internal_page_info(WebView::PageInfoType::Text); - - promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) { - test.text = text; - on_test_complete(); - }); - } else if (test.did_finish_test) { - on_test_complete(); - } - }; - - view.on_text_test_finish = [&test, on_test_complete](auto const& text) { - test.text = text; - test.did_finish_test = true; - - if (test.did_finish_loading) - on_test_complete(); - }; - } - - view.load(url); - timer->start(); -} - -static void run_ref_test(HeadlessWebContentView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds = DEFAULT_TIMEOUT_MS) -{ - auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() { - view.on_load_finish = {}; - view.on_text_test_finish = {}; - - view.on_test_complete({ test, TestResult::Timeout }); - }); - - auto handle_completed_test = [&test, url]() -> ErrorOr { - if (test.actual_screenshot->visually_equals(*test.expectation_screenshot)) - return TestResult::Pass; - - if (Application::the().dump_failed_ref_tests) { - warnln("\033[33;1mRef test {} failed; dumping screenshots\033[0m", url); - auto title = LexicalPath::title(URL::percent_decode(url.serialize_path())); - auto dump_screenshot = [&](Gfx::Bitmap& bitmap, StringView path) -> ErrorOr { - auto screenshot_file = TRY(Core::File::open(path, Core::File::OpenMode::Write)); - auto encoded_data = TRY(Gfx::PNGWriter::encode(bitmap)); - TRY(screenshot_file->write_until_depleted(encoded_data)); - warnln("\033[33;1mDumped {}\033[0m", TRY(FileSystem::real_path(path))); - return {}; - }; - - auto mkdir_result = Core::System::mkdir("test-dumps"sv, 0755); - if (mkdir_result.is_error() && mkdir_result.error().code() != EEXIST) - return mkdir_result.release_error(); - - TRY(dump_screenshot(*test.actual_screenshot, ByteString::formatted("test-dumps/{}.png", title))); - TRY(dump_screenshot(*test.expectation_screenshot, ByteString::formatted("test-dumps/{}-ref.png", title))); - } - - return TestResult::Fail; - }; - - auto on_test_complete = [&view, &test, timer, handle_completed_test]() { - timer->stop(); - - view.on_load_finish = {}; - view.on_text_test_finish = {}; - - if (auto result = handle_completed_test(); result.is_error()) - view.on_test_complete({ test, TestResult::Fail }); - else - view.on_test_complete({ test, result.value() }); - }; - - view.on_load_finish = [&view, &test, on_test_complete = move(on_test_complete)](auto const&) { - if (test.actual_screenshot) { - view.take_screenshot()->when_resolved([&test, on_test_complete = move(on_test_complete)](RefPtr screenshot) { - test.expectation_screenshot = move(screenshot); - on_test_complete(); - }); - } else { - view.take_screenshot()->when_resolved([&view, &test](RefPtr screenshot) { - test.actual_screenshot = move(screenshot); - view.debug_request("load-reference-page"); - }); - } - }; - - view.on_text_test_finish = [&](auto const&) { - dbgln("Unexpected text test finished during ref test for {}", url); - }; - - view.load(url); - timer->start(); -} - -static void run_test(HeadlessWebContentView& view, Test& test) -{ - // Clear the current document. - // FIXME: Implement a debug-request to do this more thoroughly. - auto promise = Core::Promise::construct(); - - view.on_load_finish = [promise](auto const& url) { - if (!url.equals("about:blank"sv)) - return; - - Core::deferred_invoke([promise]() { - promise->resolve({}); - }); - }; - - view.on_text_test_finish = {}; - - view.on_request_file_picker = [&](auto const& accepted_file_types, auto allow_multiple_files) { - // Create some dummy files for tests. - Vector selected_files; - - bool add_txt_files = accepted_file_types.filters.is_empty(); - bool add_cpp_files = false; - - for (auto const& filter : accepted_file_types.filters) { - filter.visit( - [](Web::HTML::FileFilter::FileType) {}, - [&](Web::HTML::FileFilter::MimeType const& mime_type) { - if (mime_type.value == "text/plain"sv) - add_txt_files = true; - }, - [&](Web::HTML::FileFilter::Extension const& extension) { - if (extension.value == "cpp"sv) - add_cpp_files = true; - }); - } - - if (add_txt_files) { - selected_files.empend("file1"sv, MUST(ByteBuffer::copy("Contents for file1"sv.bytes()))); - - if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) { - selected_files.empend("file2"sv, MUST(ByteBuffer::copy("Contents for file2"sv.bytes()))); - selected_files.empend("file3"sv, MUST(ByteBuffer::copy("Contents for file3"sv.bytes()))); - selected_files.empend("file4"sv, MUST(ByteBuffer::copy("Contents for file4"sv.bytes()))); - } - } - - if (add_cpp_files) { - selected_files.empend("file1.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 1; }}"sv.bytes()))); - - if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) { - selected_files.empend("file2.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 2; }}"sv.bytes()))); - } - } - - view.file_picker_closed(move(selected_files)); - }; - - promise->when_resolved([&view, &test](auto) { - auto url = URL::create_with_file_scheme(MUST(FileSystem::real_path(test.input_path))); - - switch (test.mode) { - case TestMode::Text: - case TestMode::Layout: - run_dump_test(view, test, url); - return; - case TestMode::Ref: - run_ref_test(view, test, url); - return; - } - - VERIFY_NOT_REACHED(); - }); - - view.load("about:blank"sv); -} - -static Vector s_skipped_tests; - -static ErrorOr load_test_config(StringView test_root_path) -{ - auto config_path = LexicalPath::join(test_root_path, "TestConfig.ini"sv); - auto config_or_error = Core::ConfigFile::open(config_path.string()); - - if (config_or_error.is_error()) { - if (config_or_error.error().code() == ENOENT) - return {}; - dbgln("Unable to open test config {}", config_path); - return config_or_error.release_error(); - } - - auto config = config_or_error.release_value(); - - for (auto const& group : config->groups()) { - if (group == "Skipped"sv) { - for (auto& key : config->keys(group)) - s_skipped_tests.append(LexicalPath::join(test_root_path, key).string()); - } else { - warnln("Unknown group '{}' in config {}", group, config_path); - } - } - return {}; -} - -static ErrorOr collect_dump_tests(Vector& tests, StringView path, StringView trail, TestMode mode) -{ - Core::DirIterator it(ByteString::formatted("{}/input/{}", path, trail), Core::DirIterator::Flags::SkipDots); - while (it.has_next()) { - auto name = it.next_path(); - auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/input/{}/{}", path, trail, name))); - if (FileSystem::is_directory(input_path)) { - TRY(collect_dump_tests(tests, path, ByteString::formatted("{}/{}", trail, name), mode)); - continue; - } - if (!name.ends_with(".html"sv) && !name.ends_with(".svg"sv)) - continue; - auto basename = LexicalPath::title(name); - auto expectation_path = ByteString::formatted("{}/expected/{}/{}.txt", path, trail, basename); - - tests.append({ mode, input_path, move(expectation_path), {} }); - } - return {}; -} - -static ErrorOr collect_ref_tests(Vector& tests, StringView path) -{ - TRY(Core::Directory::for_each_entry(path, Core::DirIterator::SkipDots, [&](Core::DirectoryEntry const& entry, Core::Directory const&) -> ErrorOr { - if (entry.type == Core::DirectoryEntry::Type::Directory) - return IterationDecision::Continue; - auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/{}", path, entry.name))); - tests.append({ TestMode::Ref, input_path, {}, {} }); - return IterationDecision::Continue; - })); - - return {}; -} - -static ErrorOr run_tests(Core::AnonymousBuffer const& theme, Gfx::IntSize window_size) -{ - auto& app = Application::the(); - TRY(load_test_config(app.test_root_path)); - - Vector tests; - auto test_glob = ByteString::formatted("*{}*", app.test_glob); - - TRY(collect_dump_tests(tests, ByteString::formatted("{}/Layout", app.test_root_path), "."sv, TestMode::Layout)); - TRY(collect_dump_tests(tests, ByteString::formatted("{}/Text", app.test_root_path), "."sv, TestMode::Text)); - TRY(collect_ref_tests(tests, ByteString::formatted("{}/Ref", app.test_root_path))); -#ifndef AK_OS_MACOS - TRY(collect_ref_tests(tests, ByteString::formatted("{}/Screenshot", app.test_root_path))); -#endif - - tests.remove_all_matching([&](auto const& test) { - return !test.input_path.matches(test_glob, CaseSensitivity::CaseSensitive); - }); - - if (app.test_dry_run) { - outln("Found {} tests...", tests.size()); - - for (auto const& [i, test] : enumerate(tests)) - outln("{}/{}: {}", i + 1, tests.size(), LexicalPath::relative_path(test.input_path, app.test_root_path)); - - return 0; - } - - auto concurrency = min(app.test_concurrency, tests.size()); - size_t loaded_web_views = 0; - - for (size_t i = 0; i < concurrency; ++i) { - auto& view = *TRY(app.create_web_view(theme, window_size)); - view.on_load_finish = [&](auto const&) { ++loaded_web_views; }; - } - - // We need to wait for the initial about:blank load to complete before starting the tests, otherwise we may load the - // test URL before the about:blank load completes. WebContent currently cannot handle this, and will drop the test URL. - Core::EventLoop::current().spin_until([&]() { - return loaded_web_views == concurrency; - }); - - size_t pass_count = 0; - size_t fail_count = 0; - size_t timeout_count = 0; - size_t skipped_count = 0; - - bool is_tty = isatty(STDOUT_FILENO); - outln("Running {} tests...", tests.size()); - - auto all_tests_complete = Core::Promise::construct(); - auto tests_remaining = tests.size(); - auto current_test = 0uz; - - Vector non_passing_tests; - - app.for_each_web_view([&](auto& view) { - view.clear_content_filters(); - - auto run_next_test = [&]() { - auto index = current_test++; - if (index >= tests.size()) - return; - auto& test = tests[index]; - - if (is_tty) { - // Keep clearing and reusing the same line if stdout is a TTY. - out("\33[2K\r"); - } - - out("{}/{}: {}", index + 1, tests.size(), LexicalPath::relative_path(test.input_path, app.test_root_path)); - - if (is_tty) - fflush(stdout); - else - outln(""); - - Core::deferred_invoke([&]() mutable { - if (s_skipped_tests.contains_slow(test.input_path)) - view.on_test_complete({ test, TestResult::Skipped }); - else - run_test(view, test); - }); - }; - - view.test_promise().when_resolved([&, run_next_test](auto result) { - switch (result.result) { - case TestResult::Pass: - ++pass_count; - break; - case TestResult::Fail: - ++fail_count; - break; - case TestResult::Timeout: - ++timeout_count; - break; - case TestResult::Skipped: - ++skipped_count; - break; - } - - if (result.result != TestResult::Pass) - non_passing_tests.append(move(result)); - - if (--tests_remaining == 0) - all_tests_complete->resolve({}); - else - run_next_test(); - }); - - Core::deferred_invoke([run_next_test]() { - run_next_test(); - }); - }); - - MUST(all_tests_complete->await()); - - if (is_tty) - outln("\33[2K\rDone!"); - - outln("=================================================="); - outln("Pass: {}, Fail: {}, Skipped: {}, Timeout: {}", pass_count, fail_count, skipped_count, timeout_count); - outln("=================================================="); - - for (auto const& non_passing_test : non_passing_tests) - outln("{}: {}", test_result_to_string(non_passing_test.result), non_passing_test.test.input_path); - - if (app.dump_gc_graph) { - app.for_each_web_view([&](auto& view) { - if (auto path = view.dump_gc_graph(); path.is_error()) - warnln("Failed to dump GC graph: {}", path.error()); - else - outln("GC graph dumped to {}", path.value()); - }); - } - - app.destroy_web_views(); - - if (timeout_count == 0 && fail_count == 0) - return 0; - return 1; -} - ErrorOr serenity_main(Main::Arguments arguments) { platform_init(); - auto app = Application::create(arguments, "about:newtab"sv); + auto app = Ladybird::Application::create(arguments, "about:newtab"sv); TRY(app->launch_services()); Core::ResourceImplementation::install(make(MUST(String::from_byte_string(app->resources_folder)))); @@ -813,7 +76,9 @@ ErrorOr serenity_main(Main::Arguments arguments) if (!app->test_root_path.is_empty()) { app->test_root_path = LexicalPath::absolute_path(TRY(FileSystem::current_working_directory()), app->test_root_path); - return run_tests(theme, window_size); + TRY(Ladybird::run_tests(theme, window_size)); + + return 0; } auto& view = *TRY(app->create_web_view(move(theme), window_size)); @@ -826,11 +91,11 @@ ErrorOr serenity_main(Main::Arguments arguments) } if (app->dump_layout_tree || app->dump_text) { - Test test { app->dump_layout_tree ? TestMode::Layout : TestMode::Text }; - run_dump_test(view, test, url); + Ladybird::Test test { app->dump_layout_tree ? Ladybird::TestMode::Layout : Ladybird::TestMode::Text }; + Ladybird::run_dump_test(view, test, url); auto completion = MUST(view.test_promise().await()); - return completion.result == TestResult::Pass ? 0 : 1; + return completion.result == Ladybird::TestResult::Pass ? 0 : 1; } if (!WebView::Application::chrome_options().webdriver_content_ipc_path.has_value()) {