diff --git a/Userland/Libraries/LibWeb/Loader/LoadRequest.cpp b/Userland/Libraries/LibWeb/Loader/LoadRequest.cpp index 8902427d3ba..434f6b05650 100644 --- a/Userland/Libraries/LibWeb/Loader/LoadRequest.cpp +++ b/Userland/Libraries/LibWeb/Loader/LoadRequest.cpp @@ -10,6 +10,13 @@ namespace Web { +static int s_resource_id = 0; + +LoadRequest::LoadRequest() + : m_id(s_resource_id++) +{ +} + LoadRequest LoadRequest::create_for_url_on_page(const URL::URL& url, Page* page) { LoadRequest request; diff --git a/Userland/Libraries/LibWeb/Loader/LoadRequest.h b/Userland/Libraries/LibWeb/Loader/LoadRequest.h index 670b43215b7..08d78b223b6 100644 --- a/Userland/Libraries/LibWeb/Loader/LoadRequest.h +++ b/Userland/Libraries/LibWeb/Loader/LoadRequest.h @@ -18,9 +18,7 @@ namespace Web { class LoadRequest { public: - LoadRequest() - { - } + LoadRequest(); static LoadRequest create_for_url_on_page(const URL::URL& url, Page* page); @@ -31,6 +29,8 @@ public: bool is_valid() const { return m_url.is_valid(); } + int id() const { return m_id; } + const URL::URL& url() const { return m_url; } void set_url(const URL::URL& url) { m_url = url; } @@ -43,7 +43,7 @@ public: void start_timer() { m_load_timer.start(); } Duration load_time() const { return m_load_timer.elapsed_time(); } - JS::GCPtr page() { return m_page.ptr(); } + JS::GCPtr page() const { return m_page.ptr(); } void set_page(Page& page) { m_page = page; } unsigned hash() const @@ -74,6 +74,7 @@ public: HashMap const& headers() const { return m_headers; } private: + int m_id { 0 }; URL::URL m_url; ByteString m_method { "GET" }; HashMap m_headers; diff --git a/Userland/Libraries/LibWeb/Loader/ResourceLoader.cpp b/Userland/Libraries/LibWeb/Loader/ResourceLoader.cpp index 9b1d171446e..b08593a776f 100644 --- a/Userland/Libraries/LibWeb/Loader/ResourceLoader.cpp +++ b/Userland/Libraries/LibWeb/Loader/ResourceLoader.cpp @@ -166,8 +166,6 @@ static void store_response_cookies(Page& page, URL::URL const& url, ByteString c } } -static size_t resource_id = 0; - static HashMap response_headers_for_file(StringView path, Optional const& modified_time) { // For file:// and resource:// URLs, we have to guess the MIME type, since there's no HTTP header to tell us what @@ -185,29 +183,72 @@ static HashMap response_hea return response_headers; } +static void log_request_start(LoadRequest const& request) +{ + auto url_for_logging = sanitized_url_for_logging(request.url()); + + emit_signpost(ByteString::formatted("Starting load: {}", url_for_logging), request.id()); + dbgln_if(SPAM_DEBUG, "ResourceLoader: Starting load of: \"{}\"", url_for_logging); +} + +static void log_success(LoadRequest const& request) +{ + auto url_for_logging = sanitized_url_for_logging(request.url()); + auto load_time_ms = request.load_time().to_milliseconds(); + + emit_signpost(ByteString::formatted("Finished load: {}", url_for_logging), request.id()); + dbgln_if(SPAM_DEBUG, "ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms); +} + +template +static void log_failure(LoadRequest const& request, ErrorType const& error) +{ + auto url_for_logging = sanitized_url_for_logging(request.url()); + auto load_time_ms = request.load_time().to_milliseconds(); + + emit_signpost(ByteString::formatted("Failed load: {}", url_for_logging), request.id()); + dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error, load_time_ms); +} + +static bool should_block_request(LoadRequest const& request) +{ + auto const& url = request.url(); + + auto is_port_blocked = [](int port) { + static constexpr auto ports = to_array({ 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, + 43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 139, + 143, 179, 389, 465, 512, 513, 514, 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636, + 993, 995, 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669 }); + + return ports.first_index_of(port).has_value(); + }; + + if (is_port_blocked(url.port_or_default())) { + log_failure(request, ByteString::formatted("Port #{} is blocked", url.port_or_default())); + return true; + } + + if (ContentFilter::the().is_filtered(url)) { + log_failure(request, "URL was filtered"sv); + return true; + } + + return false; +} + void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback, ErrorCallback error_callback, Optional timeout, TimeoutCallback timeout_callback) { - auto& url = request.url(); + auto const& url = request.url(); + + log_request_start(request); request.start_timer(); - auto id = resource_id++; - auto url_for_logging = sanitized_url_for_logging(url); - emit_signpost(ByteString::formatted("Starting load: {}", url_for_logging), id); - dbgln_if(SPAM_DEBUG, "ResourceLoader: Starting load of: \"{}\"", url_for_logging); + if (should_block_request(request)) { + error_callback("Request was blocked", {}, {}, {}); + return; + } - auto const log_success = [url_for_logging, id](auto const& request) { - auto load_time_ms = request.load_time().to_milliseconds(); - emit_signpost(ByteString::formatted("Finished load: {}", url_for_logging), id); - dbgln_if(SPAM_DEBUG, "ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms); - }; - - auto const log_failure = [url_for_logging, id](auto const& request, auto const& error_message) { - auto load_time_ms = request.load_time().to_milliseconds(); - emit_signpost(ByteString::formatted("Failed load: {}", url_for_logging), id); - dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error_message, load_time_ms); - }; - - auto respond_directory_page = [log_success, log_failure](LoadRequest const& request, URL::URL const& url, SuccessCallback const& success_callback, ErrorCallback const& error_callback) { + auto respond_directory_page = [](LoadRequest const& request, URL::URL const& url, SuccessCallback const& success_callback, ErrorCallback const& error_callback) { auto maybe_response = load_file_directory_page(url); if (maybe_response.is_error()) { log_failure(request, maybe_response.error()); @@ -222,18 +263,6 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback success_callback(maybe_response.release_value().bytes(), response_headers, {}); }; - if (is_port_blocked(url.port_or_default())) { - log_failure(request, ByteString::formatted("The port #{} is blocked", url.port_or_default())); - return; - } - - if (ContentFilter::the().is_filtered(url)) { - auto filter_message = "URL was filtered"sv; - log_failure(request, filter_message); - error_callback(filter_message, {}, {}, {}); - return; - } - if (url.scheme() == "about") { dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url); log_success(request); @@ -317,7 +346,7 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback return; } - FileRequest file_request(url.serialize_path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, respond_directory_page](ErrorOr file_or_error) { + FileRequest file_request(url.serialize_path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), request, respond_directory_page](ErrorOr file_or_error) { --m_pending_loads; if (on_load_counter_change) on_load_counter_change(); @@ -383,22 +412,10 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback } if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "gemini") { - auto proxy = ProxyMappings::the().proxy_for_url(url); - - HashMap headers; - headers.set("User-Agent", m_user_agent.to_byte_string()); - headers.set("Accept-Encoding", "gzip, deflate, br"); - - for (auto& it : request.headers()) { - headers.set(it.key, it.value); - } - - auto protocol_request = m_connector->start_request(request.method(), url, headers, request.body(), proxy); + auto protocol_request = start_network_request(request); if (!protocol_request) { - auto start_request_failure_msg = "Failed to initiate load"sv; - log_failure(request, start_request_failure_msg); if (error_callback) - error_callback(start_request_failure_msg, {}, {}, {}); + error_callback("Failed to start network request"sv, {}, {}, {}); return; } @@ -412,22 +429,9 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback timer->start(); } - m_active_requests.set(*protocol_request); - - auto on_buffered_request_finished = [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable { - --m_pending_loads; - if (on_load_counter_change) - on_load_counter_change(); - - if (request.page()) { - if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value()) - store_response_cookies(*request.page(), request.url(), *set_cookie); - if (auto cache_control = response_headers.get("cache-control"); cache_control.has_value()) { - if (cache_control.value().contains("no-store"sv)) { - s_resource_cache.remove(request); - } - } - } + auto on_buffered_request_finished = [this, success_callback = move(success_callback), error_callback = move(error_callback), request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable { + handle_network_response_headers(request, response_headers); + finish_network_request(protocol_request); if (!success || (status_code.has_value() && *status_code >= 400 && *status_code <= 599 && (payload.is_empty() || !request.is_main_resource()))) { StringBuilder error_builder; @@ -440,23 +444,12 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback error_callback(error_builder.to_byte_string(), status_code, payload, response_headers); return; } + log_success(request); success_callback(payload, response_headers, status_code); - Platform::EventLoopPlugin::the().deferred_invoke([this, &protocol_request] { - m_active_requests.remove(protocol_request); - }); }; protocol_request->set_buffered_request_finished_callback(move(on_buffered_request_finished)); - - protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey { - return {}; - }; - - ++m_pending_loads; - if (on_load_counter_change) - on_load_counter_change(); - return; } @@ -466,17 +459,107 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback error_callback(not_implemented_error, {}, {}, {}); } -bool ResourceLoader::is_port_blocked(int port) +void ResourceLoader::load_unbuffered(LoadRequest& request, OnHeadersReceived on_headers_received, OnDataReceived on_data_received, OnComplete on_complete) { - int ports[] { 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, - 43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, - 115, 117, 119, 123, 135, 139, 143, 179, 389, 465, 512, 513, 514, - 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636, 993, 995, - 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669 }; - for (auto blocked_port : ports) - if (port == blocked_port) - return true; - return false; + auto const& url = request.url(); + + log_request_start(request); + request.start_timer(); + + if (should_block_request(request)) { + on_complete(false, "Request was blocked"sv); + return; + } + + if (!url.scheme().is_one_of("http"sv, "https"sv, "gemini"sv)) { + // FIXME: Non-network requests from fetch should not go through this path. + on_complete(false, "Cannot establish connection non-network scheme"sv); + return; + } + + auto protocol_request = start_network_request(request); + if (!protocol_request) { + on_complete(false, "Failed to start network request"sv); + return; + } + + auto protocol_headers_received = [this, on_headers_received = move(on_headers_received), request](auto const& response_headers, auto status_code) { + handle_network_response_headers(request, response_headers); + on_headers_received(response_headers, move(status_code)); + }; + + auto protocol_data_received = [on_data_received = move(on_data_received)](auto data) { + on_data_received(data); + }; + + auto protocol_complete = [this, on_complete = move(on_complete), request, &protocol_request = *protocol_request](bool success, u64) { + finish_network_request(protocol_request); + + if (success) { + log_success(request); + on_complete(true, {}); + } else { + log_failure(request, "Request finished with error"sv); + on_complete(false, "Request finished with error"sv); + } + }; + + protocol_request->set_unbuffered_request_callbacks(move(protocol_headers_received), move(protocol_data_received), move(protocol_complete)); +} + +RefPtr ResourceLoader::start_network_request(LoadRequest const& request) +{ + auto proxy = ProxyMappings::the().proxy_for_url(request.url()); + + HashMap headers; + headers.set("User-Agent", m_user_agent.to_byte_string()); + headers.set("Accept-Encoding", "gzip, deflate, br"); + + for (auto const& it : request.headers()) { + headers.set(it.key, it.value); + } + + auto protocol_request = m_connector->start_request(request.method(), request.url(), headers, request.body(), proxy); + if (!protocol_request) { + log_failure(request, "Failed to initiate load"sv); + return nullptr; + } + + protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey { + return {}; + }; + + ++m_pending_loads; + if (on_load_counter_change) + on_load_counter_change(); + + m_active_requests.set(*protocol_request); + return protocol_request; +} + +void ResourceLoader::handle_network_response_headers(LoadRequest const& request, HashMap const& response_headers) +{ + if (!request.page()) + return; + + if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value()) + store_response_cookies(*request.page(), request.url(), *set_cookie); + + if (auto cache_control = response_headers.get("Cache-Control"); cache_control.has_value()) { + if (cache_control.value().contains("no-store"sv)) + s_resource_cache.remove(request); + } +} + +void ResourceLoader::finish_network_request(NonnullRefPtr const& protocol_request) +{ + --m_pending_loads; + if (on_load_counter_change) + on_load_counter_change(); + + Platform::EventLoopPlugin::the().deferred_invoke([this, protocol_request] { + m_active_requests.remove(protocol_request); + }); } void ResourceLoader::clear_cache() diff --git a/Userland/Libraries/LibWeb/Loader/ResourceLoader.h b/Userland/Libraries/LibWeb/Loader/ResourceLoader.h index 9054ba8ef49..f8423427537 100644 --- a/Userland/Libraries/LibWeb/Loader/ResourceLoader.h +++ b/Userland/Libraries/LibWeb/Loader/ResourceLoader.h @@ -78,6 +78,12 @@ public: void load(LoadRequest&, SuccessCallback success_callback, ErrorCallback error_callback = nullptr, Optional timeout = {}, TimeoutCallback timeout_callback = nullptr); + using OnHeadersReceived = JS::SafeFunction const& response_headers, Optional status_code)>; + using OnDataReceived = JS::SafeFunction; + using OnComplete = JS::SafeFunction error_message)>; + + void load_unbuffered(LoadRequest&, OnHeadersReceived, OnDataReceived, OnComplete); + ResourceLoaderConnector& connector() { return *m_connector; } void prefetch_dns(URL::URL const&); @@ -100,7 +106,9 @@ private: ResourceLoader(NonnullRefPtr); static ErrorOr> try_create(NonnullRefPtr); - static bool is_port_blocked(int port); + RefPtr start_network_request(LoadRequest const&); + void handle_network_response_headers(LoadRequest const&, HashMap const&); + void finish_network_request(NonnullRefPtr const&); int m_pending_loads { 0 };