LibWebView+WebDriver+UI: Migrate headless browsing to main Ladybird exe

We currently create a separate headless-browser application to serve two
purposes:

1. Allow headless browsing to take a screenshot of a page or print its
   layout tree / internal text.
2. Run the LibWeb test framework.

This patch migrates (1) to the main Ladybird executable. The --headless
flag enables this mode. This matches the behavior of other browsers, and
means we have one less executable to ship at distribution time.

We want to avoid creating too many AppKit / Qt facilities in headless
mode. So this involves some shuffling of application init to ensure we
don't create them until after we've parsed the command line arguments.
Namely, we avoid creating the NSApp in AppKit and QCoreApplication in
Qt. Doing so also requires that we don't create the application event
loop until we've parsed the command line as well, because the loop we
create depends on whether we're creating those UI facilities.
This commit is contained in:
Timothy Flynn 2025-06-06 15:42:12 -04:00 committed by Tim Flynn
commit c011dc766f
Notes: github-actions[bot] 2025-06-10 16:06:04 +00:00
28 changed files with 535 additions and 494 deletions

View file

@ -17,9 +17,11 @@
#include <LibWebView/Application.h>
#include <LibWebView/CookieJar.h>
#include <LibWebView/Database.h>
#include <LibWebView/HeadlessWebView.h>
#include <LibWebView/HelperProcess.h>
#include <LibWebView/URL.h>
#include <LibWebView/UserAgent.h>
#include <LibWebView/Utilities.h>
#include <LibWebView/WebContentClient.h>
namespace WebView {
@ -91,6 +93,9 @@ void Application::initialize(Main::Arguments const& arguments)
Vector<ByteString> raw_urls;
Vector<ByteString> certificates;
Optional<HeadlessMode> headless_mode;
Optional<int> window_width;
Optional<int> window_height;
bool new_window = false;
bool force_new_process = false;
bool allow_popups = false;
@ -104,6 +109,7 @@ void Application::initialize(Main::Arguments const& arguments)
Optional<StringView> dns_server_address;
Optional<u16> dns_server_port;
bool use_dns_over_tls = false;
bool layout_test_mode = false;
bool log_all_js_exceptions = false;
bool disable_site_isolation = false;
bool enable_idl_tracing = false;
@ -118,6 +124,29 @@ void Application::initialize(Main::Arguments const& arguments)
Core::ArgsParser args_parser;
args_parser.set_general_help("The Ladybird web browser :^)");
args_parser.add_positional_argument(raw_urls, "URLs to open", "url", Core::ArgsParser::Required::No);
args_parser.add_option(Core::ArgsParser::Option {
.argument_mode = Core::ArgsParser::OptionArgumentMode::Optional,
.help_string = "Run Ladybird without a browser window. Mode may be 'screenshot' (default), 'layout-tree', or 'text'.",
.long_name = "headless",
.value_name = "mode",
.accept_value = [&](StringView value) {
if (headless_mode.has_value())
return false;
if (value.is_empty() || value.equals_ignoring_ascii_case("screenshot"sv))
headless_mode = HeadlessMode::Screenshot;
else if (value.equals_ignoring_ascii_case("layout-tree"sv))
headless_mode = HeadlessMode::LayoutTree;
else if (value.equals_ignoring_ascii_case("text"sv))
headless_mode = HeadlessMode::Text;
return headless_mode.has_value();
},
});
args_parser.add_option(window_width, "Set viewport width in pixels (default: 800) (currently only supported for headless mode)", "window-width", 0, "pixels");
args_parser.add_option(window_height, "Set viewport height in pixels (default: 600) (currently only supported for headless mode)", "window-height", 0, "pixels");
args_parser.add_option(certificates, "Path to a certificate file", "certificate", 'C', "certificate");
args_parser.add_option(new_window, "Force opening in a new window", "new-window", 'n');
args_parser.add_option(force_new_process, "Force creation of a new browser process", "force-new-process");
@ -128,6 +157,7 @@ void Application::initialize(Main::Arguments const& arguments)
args_parser.add_option(profile_process, "Enable callgrind profiling of the given process name (WebContent, RequestServer, etc.)", "profile-process", 0, "process-name");
args_parser.add_option(webdriver_content_ipc_path, "Path to WebDriver IPC for WebContent", "webdriver-content-path", 0, "path", Core::ArgsParser::OptionHideMode::CommandLineAndMarkdown);
args_parser.add_option(devtools_port, "Set the Firefox DevTools server port ", "devtools-port", 0, "port");
args_parser.add_option(layout_test_mode, "Enable layout test mode", "layout-test-mode");
args_parser.add_option(log_all_js_exceptions, "Log all JavaScript exceptions", "log-all-js-exceptions");
args_parser.add_option(disable_site_isolation, "Disable site isolation", "disable-site-isolation");
args_parser.add_option(enable_idl_tracing, "Enable IDL tracing", "enable-idl-tracing");
@ -141,6 +171,7 @@ void Application::initialize(Main::Arguments const& arguments)
args_parser.add_option(dns_server_address, "Set the DNS server address", "dns-server", 0, "host|address");
args_parser.add_option(dns_server_port, "Set the DNS server port", "dns-port", 0, "port (default: 53 or 853 if --dot)");
args_parser.add_option(use_dns_over_tls, "Use DNS over TLS", "dot");
args_parser.add_option(Core::ArgsParser::Option {
.argument_mode = Core::ArgsParser::OptionArgumentMode::Required,
.help_string = "Name of the User-Agent preset to use in place of the default User-Agent",
@ -178,6 +209,7 @@ void Application::initialize(Main::Arguments const& arguments)
m_browser_options = {
.urls = sanitize_urls(raw_urls, m_settings.new_tab_page_url()),
.raw_urls = move(raw_urls),
.headless_mode = headless_mode,
.certificates = move(certificates),
.new_window = new_window ? NewWindow::Yes : NewWindow::No,
.force_new_process = force_new_process ? ForceNewProcess::Yes : ForceNewProcess::No,
@ -194,6 +226,11 @@ void Application::initialize(Main::Arguments const& arguments)
.devtools_port = devtools_port,
};
if (window_width.has_value())
m_browser_options.window_width = *window_width;
if (window_height.has_value())
m_browser_options.window_height = *window_height;
if (webdriver_content_ipc_path.has_value())
m_browser_options.webdriver_content_ipc_path = *webdriver_content_ipc_path;
@ -201,6 +238,7 @@ void Application::initialize(Main::Arguments const& arguments)
.command_line = MUST(String::join(' ', arguments.strings)),
.executable_path = MUST(String::from_byte_string(MUST(Core::System::current_executable_path()))),
.user_agent_preset = move(user_agent_preset),
.is_layout_test_mode = layout_test_mode ? IsLayoutTestMode::Yes : IsLayoutTestMode::No,
.log_all_js_exceptions = log_all_js_exceptions ? LogAllJSExceptions::Yes : LogAllJSExceptions::No,
.disable_site_isolation = disable_site_isolation ? DisableSiteIsolation::Yes : DisableSiteIsolation::No,
.enable_idl_tracing = enable_idl_tracing ? EnableIDLTracing::Yes : EnableIDLTracing::No,
@ -215,12 +253,7 @@ void Application::initialize(Main::Arguments const& arguments)
create_platform_options(m_browser_options, m_web_content_options);
if (m_browser_options.disable_sql_database == DisableSQLDatabase::No) {
m_database = Database::create().release_value_but_fixme_should_propagate_errors();
m_cookie_jar = CookieJar::create(*m_database).release_value_but_fixme_should_propagate_errors();
} else {
m_cookie_jar = CookieJar::create();
}
m_event_loop = create_platform_event_loop();
}
static ErrorOr<NonnullRefPtr<WebContentClient>> create_web_content_client(Optional<ViewImplementation&> view)
@ -278,6 +311,13 @@ void Application::launch_spare_web_content_process()
ErrorOr<void> Application::launch_services()
{
if (m_browser_options.disable_sql_database == DisableSQLDatabase::No) {
m_database = Database::create().release_value_but_fixme_should_propagate_errors();
m_cookie_jar = CookieJar::create(*m_database).release_value_but_fixme_should_propagate_errors();
} else {
m_cookie_jar = CookieJar::create();
}
TRY(launch_request_server());
TRY(launch_image_decoder_server());
return {};
@ -329,13 +369,86 @@ ErrorOr<void> Application::launch_devtools_server()
return {};
}
int Application::execute()
static NonnullRefPtr<Core::Timer> load_page_for_screenshot_and_exit(Core::EventLoop& event_loop, HeadlessWebView& view, URL::URL const& url, int screenshot_timeout)
{
int ret = m_event_loop.exec();
outln("Taking screenshot after {} seconds", screenshot_timeout);
auto timer = Core::Timer::create_single_shot(
screenshot_timeout * 1000,
[&]() {
view.take_screenshot(ViewImplementation::ScreenshotType::Full)
->when_resolved([&event_loop](auto const& path) {
outln("Saved screenshot to: {}", path);
event_loop.quit(0);
})
.when_rejected([&event_loop](auto const& error) {
warnln("Unable to take screenshot: {}", error);
event_loop.quit(0);
});
});
view.load(url);
timer->start();
return timer;
}
static void load_page_for_info_and_exit(Core::EventLoop& event_loop, HeadlessWebView& view, URL::URL const& url, WebView::PageInfoType type)
{
view.on_load_finish = [&view, &event_loop, url, type](auto const& loaded_url) {
if (!url.equals(loaded_url, URL::ExcludeFragment::Yes))
return;
view.request_internal_page_info(type)->when_resolved([&event_loop](auto const& text) {
outln("{}", text);
event_loop.quit(0);
});
};
view.load(url);
}
ErrorOr<int> Application::execute()
{
OwnPtr<HeadlessWebView> view;
RefPtr<Core::Timer> screenshot_timer;
if (m_browser_options.headless_mode.has_value()) {
auto theme_path = LexicalPath::join(WebView::s_ladybird_resource_root, "themes"sv, "Default.ini"sv);
auto theme = TRY(Gfx::load_system_theme(theme_path.string()));
view = HeadlessWebView::create(move(theme), { m_browser_options.window_width, m_browser_options.window_height });
if (!m_browser_options.webdriver_content_ipc_path.has_value()) {
if (m_browser_options.urls.size() != 1)
return Error::from_string_literal("Headless mode currently only supports exactly one URL");
switch (*m_browser_options.headless_mode) {
case HeadlessMode::Screenshot:
screenshot_timer = load_page_for_screenshot_and_exit(*m_event_loop, *view, m_browser_options.urls.first(), 1);
break;
case HeadlessMode::LayoutTree:
load_page_for_info_and_exit(*m_event_loop, *view, m_browser_options.urls.first(), WebView::PageInfoType::LayoutTree | WebView::PageInfoType::PaintTree);
break;
case HeadlessMode::Text:
load_page_for_info_and_exit(*m_event_loop, *view, m_browser_options.urls.first(), WebView::PageInfoType::Text);
break;
case HeadlessMode::Test:
VERIFY_NOT_REACHED();
}
}
}
int ret = m_event_loop->exec();
m_in_shutdown = true;
return ret;
}
NonnullOwnPtr<Core::EventLoop> Application::create_platform_event_loop()
{
return make<Core::EventLoop>();
}
void Application::add_child_process(WebView::Process&& process)
{
m_process_manager.add_process(move(process));

View file

@ -34,7 +34,7 @@ class Application : public DevTools::DevToolsDelegate {
public:
virtual ~Application();
int execute();
ErrorOr<int> execute();
static Application& the() { return *s_the; }
@ -50,8 +50,6 @@ public:
static ProcessManager& process_manager() { return the().m_process_manager; }
Core::EventLoop& event_loop() { return m_event_loop; }
ErrorOr<NonnullRefPtr<WebContentClient>> launch_web_content_process(ViewImplementation&);
ErrorOr<void> launch_services();
@ -88,6 +86,7 @@ protected:
virtual void create_platform_arguments(Core::ArgsParser&) { }
virtual void create_platform_options(BrowserOptions&, WebContentOptions&) { }
virtual NonnullOwnPtr<Core::EventLoop> create_platform_event_loop();
virtual Optional<ByteString> ask_user_for_download_folder() const { return {}; }
@ -149,7 +148,7 @@ private:
OwnPtr<Core::TimeZoneWatcher> m_time_zone_watcher;
Core::EventLoop m_event_loop;
OwnPtr<Core::EventLoop> m_event_loop;
ProcessManager m_process_manager;
bool m_in_shutdown { false };

View file

@ -9,6 +9,7 @@ set(SOURCES
CookieJar.cpp
Database.cpp
DOMNodeProperties.cpp
HeadlessWebView.cpp
HelperProcess.cpp
Mutation.cpp
Plugins/FontPlugin.cpp

View file

@ -0,0 +1,177 @@
/*
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibWebView/HeadlessWebView.h>
namespace WebView {
static Web::DevicePixelRect const screen_rect { 0, 0, 1920, 1080 };
NonnullOwnPtr<HeadlessWebView> HeadlessWebView::create(Core::AnonymousBuffer theme, Web::DevicePixelSize window_size)
{
auto view = adopt_own(*new HeadlessWebView(move(theme), window_size));
view->initialize_client(CreateNewClient::Yes);
return view;
}
NonnullOwnPtr<HeadlessWebView> HeadlessWebView::create_child(HeadlessWebView& parent, u64 page_index)
{
auto view = adopt_own(*new HeadlessWebView(parent.m_theme, parent.m_viewport_size));
view->m_client_state.client = parent.client();
view->m_client_state.page_index = page_index;
view->initialize_client(CreateNewClient::No);
return view;
}
HeadlessWebView::HeadlessWebView(Core::AnonymousBuffer theme, Web::DevicePixelSize viewport_size)
: m_theme(move(theme))
, m_viewport_size(viewport_size)
{
on_new_web_view = [this](auto, auto, Optional<u64> page_index) {
auto web_view = page_index.has_value()
? HeadlessWebView::create_child(*this, *page_index)
: HeadlessWebView::create(m_theme, m_viewport_size);
m_child_web_views.append(move(web_view));
return m_child_web_views.last()->handle();
};
on_reposition_window = [this](auto position) {
client().async_set_window_position(m_client_state.page_index, position.template to_type<Web::DevicePixels>());
client().async_did_update_window_rect(m_client_state.page_index);
};
on_resize_window = [this](auto size) {
m_viewport_size = size.template to_type<Web::DevicePixels>();
client().async_set_window_size(m_client_state.page_index, m_viewport_size);
client().async_set_viewport_size(m_client_state.page_index, m_viewport_size);
client().async_did_update_window_rect(m_client_state.page_index);
};
on_restore_window = [this]() {
set_system_visibility_state(Web::HTML::VisibilityState::Visible);
};
on_minimize_window = [this]() {
set_system_visibility_state(Web::HTML::VisibilityState::Hidden);
};
on_maximize_window = [this]() {
m_viewport_size = screen_rect.size();
client().async_set_window_position(m_client_state.page_index, screen_rect.location());
client().async_set_window_size(m_client_state.page_index, screen_rect.size());
client().async_set_viewport_size(m_client_state.page_index, screen_rect.size());
client().async_did_update_window_rect(m_client_state.page_index);
};
on_fullscreen_window = [this]() {
m_viewport_size = screen_rect.size();
client().async_set_window_position(m_client_state.page_index, screen_rect.location());
client().async_set_window_size(m_client_state.page_index, screen_rect.size());
client().async_set_viewport_size(m_client_state.page_index, screen_rect.size());
client().async_did_update_window_rect(m_client_state.page_index);
};
on_request_alert = [this](auto const&) {
m_pending_dialog = Web::Page::PendingDialog::Alert;
};
on_request_confirm = [this](auto const&) {
m_pending_dialog = Web::Page::PendingDialog::Confirm;
};
on_request_prompt = [this](auto const&, auto const& prompt_text) {
m_pending_dialog = Web::Page::PendingDialog::Prompt;
m_pending_prompt_text = prompt_text;
};
on_request_set_prompt_text = [this](auto const& prompt_text) {
m_pending_prompt_text = prompt_text;
};
on_request_accept_dialog = [this]() {
switch (m_pending_dialog) {
case Web::Page::PendingDialog::None:
VERIFY_NOT_REACHED();
break;
case Web::Page::PendingDialog::Alert:
alert_closed();
break;
case Web::Page::PendingDialog::Confirm:
confirm_closed(true);
break;
case Web::Page::PendingDialog::Prompt:
prompt_closed(move(m_pending_prompt_text));
break;
}
m_pending_dialog = Web::Page::PendingDialog::None;
};
on_request_dismiss_dialog = [this]() {
switch (m_pending_dialog) {
case Web::Page::PendingDialog::None:
VERIFY_NOT_REACHED();
break;
case Web::Page::PendingDialog::Alert:
alert_closed();
break;
case Web::Page::PendingDialog::Confirm:
confirm_closed(false);
break;
case Web::Page::PendingDialog::Prompt:
prompt_closed({});
break;
}
m_pending_dialog = Web::Page::PendingDialog::None;
m_pending_prompt_text.clear();
};
on_insert_clipboard_entry = [this](Web::Clipboard::SystemClipboardRepresentation entry, auto const&) {
Web::Clipboard::SystemClipboardItem item;
item.system_clipboard_representations.append(move(entry));
m_clipboard = move(item);
};
on_request_clipboard_entries = [this](auto request_id) {
if (m_clipboard.has_value())
retrieved_clipboard_entries(request_id, { { *m_clipboard } });
else
retrieved_clipboard_entries(request_id, {});
};
m_system_visibility_state = Web::HTML::VisibilityState::Visible;
}
void HeadlessWebView::initialize_client(CreateNewClient create_new_client)
{
ViewImplementation::initialize_client(create_new_client);
client().async_update_system_theme(m_client_state.page_index, m_theme);
client().async_set_viewport_size(m_client_state.page_index, viewport_size());
client().async_set_window_size(m_client_state.page_index, viewport_size());
client().async_update_screen_rects(m_client_state.page_index, { { screen_rect } }, 0);
}
void HeadlessWebView::update_zoom()
{
client().async_set_device_pixels_per_css_pixel(m_client_state.page_index, m_device_pixel_ratio * m_zoom_level);
client().async_set_viewport_size(m_client_state.page_index, m_viewport_size);
}
}

View file

@ -0,0 +1,44 @@
/*
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibCore/Forward.h>
#include <LibGfx/Forward.h>
#include <LibWeb/Page/Page.h>
#include <LibWeb/PixelUnits.h>
#include <LibWebView/ViewImplementation.h>
namespace WebView {
class HeadlessWebView : public WebView::ViewImplementation {
public:
static NonnullOwnPtr<HeadlessWebView> create(Core::AnonymousBuffer theme, Web::DevicePixelSize window_size);
static NonnullOwnPtr<HeadlessWebView> create_child(HeadlessWebView&, u64 page_index);
protected:
HeadlessWebView(Core::AnonymousBuffer theme, Web::DevicePixelSize viewport_size);
void initialize_client(CreateNewClient) override;
void update_zoom() override;
virtual Web::DevicePixelSize viewport_size() const override { return m_viewport_size; }
virtual Gfx::IntPoint to_content_position(Gfx::IntPoint widget_position) const override { return widget_position; }
virtual Gfx::IntPoint to_widget_position(Gfx::IntPoint content_position) const override { return content_position; }
Core::AnonymousBuffer m_theme;
Web::DevicePixelSize m_viewport_size;
Web::Page::PendingDialog m_pending_dialog { Web::Page::PendingDialog::None };
Optional<String> m_pending_prompt_text;
// FIXME: We should implement UI-agnostic platform APIs to interact with the system clipboard.
Optional<Web::Clipboard::SystemClipboardItem> m_clipboard;
Vector<NonnullOwnPtr<HeadlessWebView>> m_child_web_views;
};
}

View file

@ -85,6 +85,7 @@ static ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_proc
Optional<IPC::File> request_server_socket,
ClientArguments&&... client_arguments)
{
auto const& browser_options = WebView::Application::browser_options();
auto const& web_content_options = WebView::Application::web_content_options();
Vector<ByteString> arguments {
@ -94,6 +95,9 @@ static ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_proc
web_content_options.executable_path.to_byte_string(),
};
if (browser_options.headless_mode.has_value())
arguments.append("--headless"sv);
if (web_content_options.config_path.has_value()) {
arguments.append("--config-path"sv);
arguments.append(web_content_options.config_path.value());
@ -116,8 +120,6 @@ static ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_proc
arguments.append("--force-fontconfig"sv);
if (web_content_options.collect_garbage_on_every_allocation == WebView::CollectGarbageOnEveryAllocation::Yes)
arguments.append("--collect-garbage-on-every-allocation"sv);
if (web_content_options.is_headless == WebView::IsHeadless::Yes)
arguments.append("--headless"sv);
if (web_content_options.paint_viewport_scrollbars == PaintViewportScrollbars::No)
arguments.append("--disable-scrollbar-painting"sv);

View file

@ -15,6 +15,13 @@
namespace WebView {
enum class HeadlessMode {
Screenshot,
LayoutTree,
Text,
Test,
};
enum class NewWindow {
No,
Yes,
@ -62,6 +69,9 @@ constexpr inline u16 default_devtools_port = 6000;
struct BrowserOptions {
Vector<URL::URL> urls;
Vector<ByteString> raw_urls;
Optional<HeadlessMode> headless_mode;
int window_width { 800 };
int window_height { 600 };
Vector<ByteString> certificates {};
NewWindow new_window { NewWindow::No };
ForceNewProcess force_new_process { ForceNewProcess::No };
@ -120,11 +130,6 @@ enum class CollectGarbageOnEveryAllocation {
Yes,
};
enum class IsHeadless {
No,
Yes,
};
enum class PaintViewportScrollbars {
Yes,
No,
@ -146,7 +151,6 @@ struct WebContentOptions {
EnableAutoplay enable_autoplay { EnableAutoplay::No };
CollectGarbageOnEveryAllocation collect_garbage_on_every_allocation { CollectGarbageOnEveryAllocation::No };
Optional<u16> echo_server_port {};
IsHeadless is_headless { IsHeadless::No };
PaintViewportScrollbars paint_viewport_scrollbars { PaintViewportScrollbars::Yes };
};