LibWeb: Support unbuffered resource load requests

This adds an alternate API to ResourceLoader to load HTTP/HTTPS/Gemini
requests unbuffered. Most of the changes here are moving parts of the
existing ResourceLoader::load method to helper methods so they can be
re-used by the new ResourceLoader::load_unbuffered.

Userland/Libraries/LibWeb/Loader/LoadRequest.cpp

@@ -10,6 +10,13 @@
 
 namespace Web {
 
+static int s_resource_id = 0;
+
+LoadRequest::LoadRequest()
+    : m_id(s_resource_id++)
+{
+}
+
 LoadRequest LoadRequest::create_for_url_on_page(const URL::URL& url, Page* page)
 {
     LoadRequest request;

Userland/Libraries/LibWeb/Loader/LoadRequest.h

@@ -18,9 +18,7 @@ namespace Web {
 
 class LoadRequest {
 public:
-    LoadRequest()
-    {
-    }
+    LoadRequest();
 
     static LoadRequest create_for_url_on_page(const URL::URL& url, Page* page);
 
@@ -31,6 +29,8 @@ public:
 
     bool is_valid() const { return m_url.is_valid(); }
 
+    int id() const { return m_id; }
+
     const URL::URL& url() const { return m_url; }
     void set_url(const URL::URL& url) { m_url = url; }
 
@@ -43,7 +43,7 @@ public:
     void start_timer() { m_load_timer.start(); }
     Duration load_time() const { return m_load_timer.elapsed_time(); }
 
-    JS::GCPtr<Page> page() { return m_page.ptr(); }
+    JS::GCPtr<Page> page() const { return m_page.ptr(); }
     void set_page(Page& page) { m_page = page; }
 
     unsigned hash() const
@@ -74,6 +74,7 @@ public:
     HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& headers() const { return m_headers; }
 
 private:
+    int m_id { 0 };
     URL::URL m_url;
     ByteString m_method { "GET" };
     HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> m_headers;

Userland/Libraries/LibWeb/Loader/ResourceLoader.cpp

@@ -166,8 +166,6 @@ static void store_response_cookies(Page& page, URL::URL const& url, ByteString c
     }
 }
 
-static size_t resource_id = 0;
-
 static HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_headers_for_file(StringView path, Optional<time_t> const& modified_time)
 {
     // For file:// and resource:// URLs, we have to guess the MIME type, since there's no HTTP header to tell us what
@@ -185,29 +183,72 @@ static HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> response_hea
     return response_headers;
 }
 
+static void log_request_start(LoadRequest const& request)
+{
+    auto url_for_logging = sanitized_url_for_logging(request.url());
+
+    emit_signpost(ByteString::formatted("Starting load: {}", url_for_logging), request.id());
+    dbgln_if(SPAM_DEBUG, "ResourceLoader: Starting load of: \"{}\"", url_for_logging);
+}
+
+static void log_success(LoadRequest const& request)
+{
+    auto url_for_logging = sanitized_url_for_logging(request.url());
+    auto load_time_ms = request.load_time().to_milliseconds();
+
+    emit_signpost(ByteString::formatted("Finished load: {}", url_for_logging), request.id());
+    dbgln_if(SPAM_DEBUG, "ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms);
+}
+
+template<typename ErrorType>
+static void log_failure(LoadRequest const& request, ErrorType const& error)
+{
+    auto url_for_logging = sanitized_url_for_logging(request.url());
+    auto load_time_ms = request.load_time().to_milliseconds();
+
+    emit_signpost(ByteString::formatted("Failed load: {}", url_for_logging), request.id());
+    dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error, load_time_ms);
+}
+
+static bool should_block_request(LoadRequest const& request)
+{
+    auto const& url = request.url();
+
+    auto is_port_blocked = [](int port) {
+        static constexpr auto ports = to_array({ 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42,
+            43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 139,
+            143, 179, 389, 465, 512, 513, 514, 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636,
+            993, 995, 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669 });
+
+        return ports.first_index_of(port).has_value();
+    };
+
+    if (is_port_blocked(url.port_or_default())) {
+        log_failure(request, ByteString::formatted("Port #{} is blocked", url.port_or_default()));
+        return true;
+    }
+
+    if (ContentFilter::the().is_filtered(url)) {
+        log_failure(request, "URL was filtered"sv);
+        return true;
+    }
+
+    return false;
+}
+
 void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback, ErrorCallback error_callback, Optional<u32> timeout, TimeoutCallback timeout_callback)
 {
-    auto& url = request.url();
+    auto const& url = request.url();
+
+    log_request_start(request);
     request.start_timer();
 
-    auto id = resource_id++;
-    auto url_for_logging = sanitized_url_for_logging(url);
-    emit_signpost(ByteString::formatted("Starting load: {}", url_for_logging), id);
-    dbgln_if(SPAM_DEBUG, "ResourceLoader: Starting load of: \"{}\"", url_for_logging);
+    if (should_block_request(request)) {
+        error_callback("Request was blocked", {}, {}, {});
+        return;
+    }
 
-    auto const log_success = [url_for_logging, id](auto const& request) {
-        auto load_time_ms = request.load_time().to_milliseconds();
-        emit_signpost(ByteString::formatted("Finished load: {}", url_for_logging), id);
-        dbgln_if(SPAM_DEBUG, "ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms);
-    };
-
-    auto const log_failure = [url_for_logging, id](auto const& request, auto const& error_message) {
-        auto load_time_ms = request.load_time().to_milliseconds();
-        emit_signpost(ByteString::formatted("Failed load: {}", url_for_logging), id);
-        dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error_message, load_time_ms);
-    };
-
-    auto respond_directory_page = [log_success, log_failure](LoadRequest const& request, URL::URL const& url, SuccessCallback const& success_callback, ErrorCallback const& error_callback) {
+    auto respond_directory_page = [](LoadRequest const& request, URL::URL const& url, SuccessCallback const& success_callback, ErrorCallback const& error_callback) {
         auto maybe_response = load_file_directory_page(url);
         if (maybe_response.is_error()) {
             log_failure(request, maybe_response.error());
@@ -222,18 +263,6 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
         success_callback(maybe_response.release_value().bytes(), response_headers, {});
     };
 
-    if (is_port_blocked(url.port_or_default())) {
-        log_failure(request, ByteString::formatted("The port #{} is blocked", url.port_or_default()));
-        return;
-    }
-
-    if (ContentFilter::the().is_filtered(url)) {
-        auto filter_message = "URL was filtered"sv;
-        log_failure(request, filter_message);
-        error_callback(filter_message, {}, {}, {});
-        return;
-    }
-
     if (url.scheme() == "about") {
         dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url);
         log_success(request);
@@ -317,7 +346,7 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
             return;
         }
 
-        FileRequest file_request(url.serialize_path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, respond_directory_page](ErrorOr<i32> file_or_error) {
+        FileRequest file_request(url.serialize_path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), request, respond_directory_page](ErrorOr<i32> file_or_error) {
             --m_pending_loads;
             if (on_load_counter_change)
                 on_load_counter_change();
@@ -383,22 +412,10 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
     }
 
     if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "gemini") {
-        auto proxy = ProxyMappings::the().proxy_for_url(url);
-
-        HashMap<ByteString, ByteString> headers;
-        headers.set("User-Agent", m_user_agent.to_byte_string());
-        headers.set("Accept-Encoding", "gzip, deflate, br");
-
-        for (auto& it : request.headers()) {
-            headers.set(it.key, it.value);
-        }
-
-        auto protocol_request = m_connector->start_request(request.method(), url, headers, request.body(), proxy);
+        auto protocol_request = start_network_request(request);
         if (!protocol_request) {
-            auto start_request_failure_msg = "Failed to initiate load"sv;
-            log_failure(request, start_request_failure_msg);
             if (error_callback)
-                error_callback(start_request_failure_msg, {}, {}, {});
+                error_callback("Failed to start network request"sv, {}, {}, {});
             return;
         }
 
@@ -412,22 +429,9 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
             timer->start();
         }
 
-        m_active_requests.set(*protocol_request);
-
-        auto on_buffered_request_finished = [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable {
-            --m_pending_loads;
-            if (on_load_counter_change)
-                on_load_counter_change();
-
-            if (request.page()) {
-                if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value())
-                    store_response_cookies(*request.page(), request.url(), *set_cookie);
-                if (auto cache_control = response_headers.get("cache-control"); cache_control.has_value()) {
-                    if (cache_control.value().contains("no-store"sv)) {
-                        s_resource_cache.remove(request);
-                    }
-                }
-            }
+        auto on_buffered_request_finished = [this, success_callback = move(success_callback), error_callback = move(error_callback), request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable {
+            handle_network_response_headers(request, response_headers);
+            finish_network_request(protocol_request);
 
             if (!success || (status_code.has_value() && *status_code >= 400 && *status_code <= 599 && (payload.is_empty() || !request.is_main_resource()))) {
                 StringBuilder error_builder;
@@ -440,23 +444,12 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
                     error_callback(error_builder.to_byte_string(), status_code, payload, response_headers);
                 return;
             }
+
             log_success(request);
             success_callback(payload, response_headers, status_code);
-            Platform::EventLoopPlugin::the().deferred_invoke([this, &protocol_request] {
-                m_active_requests.remove(protocol_request);
-            });
         };
 
         protocol_request->set_buffered_request_finished_callback(move(on_buffered_request_finished));
-
-        protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey {
-            return {};
-        };
-
-        ++m_pending_loads;
-        if (on_load_counter_change)
-            on_load_counter_change();
-
         return;
     }
 
@@ -466,17 +459,107 @@ void ResourceLoader::load(LoadRequest& request, SuccessCallback success_callback
         error_callback(not_implemented_error, {}, {}, {});
 }
 
-bool ResourceLoader::is_port_blocked(int port)
+void ResourceLoader::load_unbuffered(LoadRequest& request, OnHeadersReceived on_headers_received, OnDataReceived on_data_received, OnComplete on_complete)
 {
-    int ports[] { 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42,
-        43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113,
-        115, 117, 119, 123, 135, 139, 143, 179, 389, 465, 512, 513, 514,
-        515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636, 993, 995,
-        2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669 };
-    for (auto blocked_port : ports)
-        if (port == blocked_port)
-            return true;
-    return false;
+    auto const& url = request.url();
+
+    log_request_start(request);
+    request.start_timer();
+
+    if (should_block_request(request)) {
+        on_complete(false, "Request was blocked"sv);
+        return;
+    }
+
+    if (!url.scheme().is_one_of("http"sv, "https"sv, "gemini"sv)) {
+        // FIXME: Non-network requests from fetch should not go through this path.
+        on_complete(false, "Cannot establish connection non-network scheme"sv);
+        return;
+    }
+
+    auto protocol_request = start_network_request(request);
+    if (!protocol_request) {
+        on_complete(false, "Failed to start network request"sv);
+        return;
+    }
+
+    auto protocol_headers_received = [this, on_headers_received = move(on_headers_received), request](auto const& response_headers, auto status_code) {
+        handle_network_response_headers(request, response_headers);
+        on_headers_received(response_headers, move(status_code));
+    };
+
+    auto protocol_data_received = [on_data_received = move(on_data_received)](auto data) {
+        on_data_received(data);
+    };
+
+    auto protocol_complete = [this, on_complete = move(on_complete), request, &protocol_request = *protocol_request](bool success, u64) {
+        finish_network_request(protocol_request);
+
+        if (success) {
+            log_success(request);
+            on_complete(true, {});
+        } else {
+            log_failure(request, "Request finished with error"sv);
+            on_complete(false, "Request finished with error"sv);
+        }
+    };
+
+    protocol_request->set_unbuffered_request_callbacks(move(protocol_headers_received), move(protocol_data_received), move(protocol_complete));
+}
+
+RefPtr<ResourceLoaderConnectorRequest> ResourceLoader::start_network_request(LoadRequest const& request)
+{
+    auto proxy = ProxyMappings::the().proxy_for_url(request.url());
+
+    HashMap<ByteString, ByteString> headers;
+    headers.set("User-Agent", m_user_agent.to_byte_string());
+    headers.set("Accept-Encoding", "gzip, deflate, br");
+
+    for (auto const& it : request.headers()) {
+        headers.set(it.key, it.value);
+    }
+
+    auto protocol_request = m_connector->start_request(request.method(), request.url(), headers, request.body(), proxy);
+    if (!protocol_request) {
+        log_failure(request, "Failed to initiate load"sv);
+        return nullptr;
+    }
+
+    protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey {
+        return {};
+    };
+
+    ++m_pending_loads;
+    if (on_load_counter_change)
+        on_load_counter_change();
+
+    m_active_requests.set(*protocol_request);
+    return protocol_request;
+}
+
+void ResourceLoader::handle_network_response_headers(LoadRequest const& request, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers)
+{
+    if (!request.page())
+        return;
+
+    if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value())
+        store_response_cookies(*request.page(), request.url(), *set_cookie);
+
+    if (auto cache_control = response_headers.get("Cache-Control"); cache_control.has_value()) {
+        if (cache_control.value().contains("no-store"sv))
+            s_resource_cache.remove(request);
+    }
+}
+
+void ResourceLoader::finish_network_request(NonnullRefPtr<ResourceLoaderConnectorRequest> const& protocol_request)
+{
+    --m_pending_loads;
+    if (on_load_counter_change)
+        on_load_counter_change();
+
+    Platform::EventLoopPlugin::the().deferred_invoke([this, protocol_request] {
+        m_active_requests.remove(protocol_request);
+    });
 }
 
 void ResourceLoader::clear_cache()

Userland/Libraries/LibWeb/Loader/ResourceLoader.h

@@ -78,6 +78,12 @@ public:
 
     void load(LoadRequest&, SuccessCallback success_callback, ErrorCallback error_callback = nullptr, Optional<u32> timeout = {}, TimeoutCallback timeout_callback = nullptr);
 
+    using OnHeadersReceived = JS::SafeFunction<void(HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)>;
+    using OnDataReceived = JS::SafeFunction<void(ReadonlyBytes data)>;
+    using OnComplete = JS::SafeFunction<void(bool success, Optional<StringView> error_message)>;
+
+    void load_unbuffered(LoadRequest&, OnHeadersReceived, OnDataReceived, OnComplete);
+
     ResourceLoaderConnector& connector() { return *m_connector; }
 
     void prefetch_dns(URL::URL const&);
@@ -100,7 +106,9 @@ private:
     ResourceLoader(NonnullRefPtr<ResourceLoaderConnector>);
     static ErrorOr<NonnullRefPtr<ResourceLoader>> try_create(NonnullRefPtr<ResourceLoaderConnector>);
 
-    static bool is_port_blocked(int port);
+    RefPtr<ResourceLoaderConnectorRequest> start_network_request(LoadRequest const&);
+    void handle_network_response_headers(LoadRequest const&, HashMap<ByteString, ByteString, CaseInsensitiveStringTraits> const&);
+    void finish_network_request(NonnullRefPtr<ResourceLoaderConnectorRequest> const&);
 
     int m_pending_loads { 0 };