diff --git a/Userland/Libraries/LibWebView/CMakeLists.txt b/Userland/Libraries/LibWebView/CMakeLists.txt index 38f5056e7cf..fa851a82b84 100644 --- a/Userland/Libraries/LibWebView/CMakeLists.txt +++ b/Userland/Libraries/LibWebView/CMakeLists.txt @@ -67,6 +67,4 @@ if (ENABLE_INSTALL_HEADERS) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${header}" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${subdirectory}") endforeach() - - install(FILES "${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/ConnectionCache.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/RequestServer") endif() diff --git a/Userland/Services/RequestServer/CMakeLists.txt b/Userland/Services/RequestServer/CMakeLists.txt index e4a4b01dd60..47d29faafb9 100644 --- a/Userland/Services/RequestServer/CMakeLists.txt +++ b/Userland/Services/RequestServer/CMakeLists.txt @@ -3,14 +3,8 @@ compile_ipc(RequestClient.ipc RequestClientEndpoint.h) set(SOURCES ConnectionFromClient.cpp - ConnectionCache.cpp Request.cpp - HttpRequest.cpp - HttpProtocol.cpp - HttpsRequest.cpp - HttpsProtocol.cpp main.cpp - Protocol.cpp ) set(GENERATED_SOURCES @@ -19,4 +13,4 @@ set(GENERATED_SOURCES ) serenity_bin(RequestServer) -target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibHTTP LibMain LibTLS LibWebSocket LibURL LibThreading) +target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibMain LibTLS LibWebSocket LibURL LibThreading) diff --git a/Userland/Services/RequestServer/CacheLevel.h b/Userland/Services/RequestServer/CacheLevel.h new file mode 100644 index 00000000000..2372c303554 --- /dev/null +++ b/Userland/Services/RequestServer/CacheLevel.h @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2024, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +namespace RequestServer { + +enum class CacheLevel { + ResolveOnly, + CreateConnection, +}; + +} diff --git a/Userland/Services/RequestServer/ConnectionCache.cpp b/Userland/Services/RequestServer/ConnectionCache.cpp deleted file mode 100644 index 9a105d07476..00000000000 --- a/Userland/Services/RequestServer/ConnectionCache.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2021-2022, Ali Mohammad Pur - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include "ConnectionCache.h" -#include -#include -#include - -namespace RequestServer::ConnectionCache { - -Threading::RWLockProtected>>>>> g_tcp_connection_cache {}; -Threading::RWLockProtected>>>>> g_tls_connection_cache {}; -Threading::RWLockProtected> g_inferred_server_properties; - -void request_did_finish(URL::URL const& url, Core::Socket const* socket) -{ - if (!socket) { - dbgln("Request with a null socket finished for URL {}", url); - return; - } - - dbgln_if(REQUESTSERVER_DEBUG, "Request for {} finished", url); - - ConnectionKey partial_key { url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string(), url.port_or_default() }; - auto fire_off_next_job = [&](auto& cache) { - using CacheType = typename RemoveCVReference::ProtectedType; - auto [it, end] = cache.with_read_locked([&](auto const& cache) { - struct Result { - decltype(cache.begin()) it; - decltype(cache.end()) end; - }; - return Result { - find_if(cache.begin(), cache.end(), [&](auto& connection) { - return connection.key.hostname == partial_key.hostname && connection.key.port == partial_key.port; - }), - cache.end(), - }; - }); - if (it == end) { - dbgln("Request for URL {} finished, but we don't own that!", url); - return; - } - auto connection_it = it->value->find_if([&](auto& connection) { return connection->socket == socket; }); - if (connection_it.is_end()) { - dbgln("Request for URL {} finished, but we don't have a socket for that!", url); - return; - } - - auto& connection = *connection_it; - if constexpr (REQUESTSERVER_DEBUG) { - connection->job_data->timing_info.performing_request = AK::Duration::from_milliseconds(connection->job_data->timing_info.timer.elapsed_milliseconds()); - connection->job_data->timing_info.timer.start(); - } - - auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(partial_key.hostname); }); - if (!connection->socket->is_open()) - properties.requests_served_per_connection = min(properties.requests_served_per_connection, connection->max_queue_length + 1); - - if (connection->request_queue.with_read_locked([](auto const& queue) { return queue.is_empty(); })) { - // Immediately mark the connection as finished, as new jobs will never be run if they are queued - // before the deferred_invoke() below runs otherwise. - connection->has_started = false; - connection->socket->set_notifications_enabled(false); - - Core::deferred_invoke([&connection, &cache_entry = *it->value, key = it->key, &cache] { - if (connection->has_started) - return; - - connection->current_url = {}; - connection->job_data = {}; - connection->removal_timer->on_timeout = [ptr = connection.ptr(), &cache_entry, key = move(key), &cache]() mutable { - Core::deferred_invoke([&, key = move(key), ptr] { - if (ptr->has_started) - return; - - dbgln_if(REQUESTSERVER_DEBUG, "Removing no-longer-used connection {} (socket {})", ptr, ptr->socket.ptr()); - cache.with_write_locked([&](CacheType& cache) { - auto did_remove = cache_entry.remove_first_matching([&](auto& entry) { return entry == ptr; }); - VERIFY(did_remove); - if (cache_entry.is_empty()) - cache.remove(key); - }); - }); - }; - connection->removal_timer->start(); - }); - } else { - auto timer = Core::ElapsedTimer::start_new(); - if (auto result = recreate_socket_if_needed(*connection, url); result.is_error()) { - if constexpr (REQUESTSERVER_DEBUG) { - connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds()); - } - cache.with_read_locked([&](auto&) { - dbgln("ConnectionCache request finish handler, reconnection failed with {}", result.error()); - connection->job_data->fail(Core::NetworkJob::Error::ConnectionFailed); - }); - return; - } - if constexpr (REQUESTSERVER_DEBUG) { - connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds()); - } - - connection->has_started = true; - Core::deferred_invoke([&connection = *connection, url, &cache] { - cache.with_read_locked([&](auto&) { - dbgln_if(REQUESTSERVER_DEBUG, "Running next job in queue for connection {}", &connection); - connection.timer.start(); - connection.current_url = url; - connection.job_data = connection.request_queue.with_write_locked([](auto& queue) { return queue.take_first(); }); - if constexpr (REQUESTSERVER_DEBUG) { - connection.job_data->timing_info.waiting_in_queue = AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds() - connection.job_data->timing_info.performing_request.to_milliseconds()); - connection.job_data->timing_info.timer.start(); - } - connection.socket->set_notifications_enabled(true); - connection.job_data->start(*connection.socket); - }); - }); - } - }; - - if (is>(socket)) - fire_off_next_job(g_tls_connection_cache); - else if (is>(socket)) - fire_off_next_job(g_tcp_connection_cache); - else - dbgln("Unknown socket {} finished for URL {}", socket, url); -} - -void dump_jobs() -{ - g_tls_connection_cache.with_read_locked([](auto& cache) { - dbgln("=========== TLS Connection Cache =========="); - for (auto& connection : cache) { - dbgln(" - {}:{}", connection.key.hostname, connection.key.port); - for (auto& entry : *connection.value) { - dbgln(" - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr()); - dbgln(" Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0); - dbgln(" Request Queue:"); - entry->request_queue.for_each_locked([](auto const& job) { - dbgln(" - {}", &job); - }); - } - } - }); - - g_tcp_connection_cache.with_read_locked([](auto& cache) { - dbgln("=========== TCP Connection Cache =========="); - for (auto& connection : cache) { - dbgln(" - {}:{}", connection.key.hostname, connection.key.port); - for (auto& entry : *connection.value) { - dbgln(" - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr()); - dbgln(" Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0); - dbgln(" Request Queue:"); - entry->request_queue.for_each_locked([](auto const& job) { - dbgln(" - {}", &job); - }); - } - } - }); -} - -size_t hits; -size_t misses; -} diff --git a/Userland/Services/RequestServer/ConnectionCache.h b/Userland/Services/RequestServer/ConnectionCache.h deleted file mode 100644 index e89b3470bd1..00000000000 --- a/Userland/Services/RequestServer/ConnectionCache.h +++ /dev/null @@ -1,386 +0,0 @@ -/* - * Copyright (c) 2021-2022, Ali Mohammad Pur - * Copyright (c) 2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer { - -enum class CacheLevel { - ResolveOnly, - CreateConnection, -}; - -} - -namespace RequestServer::ConnectionCache { - -struct Proxy { - Core::ProxyData data; - OwnPtr proxy_client_storage {}; - - template - ErrorOr> tunnel(URL::URL const& url, Args&&... args) - { - if (data.type == Core::ProxyData::Direct) { - return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), url.port_or_default(), forward(args)...)); - } - if (data.type == Core::ProxyData::SOCKS5) { - if constexpr (requires { SocketType::connect(declval(), *proxy_client_storage, forward(args)...); }) { - proxy_client_storage = TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default())); - return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), *proxy_client_storage, forward(args)...)); - } else if constexpr (IsSame) { - return TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default())); - } else { - return Error::from_string_literal("SOCKS5 not supported for this socket type"); - } - } - VERIFY_NOT_REACHED(); - } -}; - -struct JobData { - Function start {}; - Function fail {}; - Function()> provide_client_certificates {}; - struct TimingInfo { -#if REQUESTSERVER_DEBUG - bool valid { true }; - Core::ElapsedTimer timer {}; - URL::URL url {}; - AK::Duration waiting_in_queue {}; - AK::Duration starting_connection {}; - AK::Duration performing_request {}; -#endif - } timing_info {}; - - JobData(Function start, Function fail, Function()> provide_client_certificates, TimingInfo timing_info) - : start(move(start)) - , fail(move(fail)) - , provide_client_certificates(move(provide_client_certificates)) - , timing_info(move(timing_info)) - { - } - - JobData(JobData&& other) - : start(move(other.start)) - , fail(move(other.fail)) - , provide_client_certificates(move(other.provide_client_certificates)) - , timing_info(move(other.timing_info)) - { -#if REQUESTSERVER_DEBUG - other.timing_info.valid = false; -#endif - } - -#if REQUESTSERVER_DEBUG - ~JobData() - { - if (timing_info.valid) { - dbgln("JobData for {} timings:", timing_info.url); - dbgln(" - Waiting in queue: {}ms", timing_info.waiting_in_queue.to_milliseconds()); - dbgln(" - Starting connection: {}ms", timing_info.starting_connection.to_milliseconds()); - dbgln(" - Performing request: {}ms", timing_info.performing_request.to_milliseconds()); - } - } -#endif - - static JobData create(auto job, [[maybe_unused]] URL::URL url) - { - return JobData { - [job](auto& socket) { job->start(socket); }, - [job](auto error) { job->fail(error); }, - [job] { - if constexpr (requires { job->on_certificate_requested; }) { - if (job->on_certificate_requested) - return job->on_certificate_requested(); - } else { - // "use" `job`, otherwise clang gets sad. - (void)job; - } - return Vector {}; - }, - { -#if REQUESTSERVER_DEBUG - .timer = Core::ElapsedTimer::start_new(Core::TimerType::Precise), - .url = move(url), - .waiting_in_queue = {}, - .starting_connection = {}, - .performing_request = {}, -#endif - }, - }; - } -}; - -template -struct Connection { - using QueueType = Vector; - using SocketType = Socket; - using StorageType = SocketStorageType; - - OwnPtr> socket; - Threading::RWLockProtected request_queue; - NonnullRefPtr removal_timer; - Atomic is_being_started { false }; - bool has_started { false }; - URL::URL current_url {}; - Core::ElapsedTimer timer {}; - Optional job_data {}; - Proxy proxy {}; - size_t max_queue_length { 0 }; -}; - -struct ConnectionKey { - ByteString hostname; - u16 port { 0 }; - Core::ProxyData proxy_data {}; - - bool operator==(ConnectionKey const&) const = default; -}; - -}; - -template<> -struct AK::Traits : public AK::DefaultTraits { - static u32 hash(RequestServer::ConnectionCache::ConnectionKey const& key) - { - return pair_int_hash(pair_int_hash(key.proxy_data.host_ipv4, key.proxy_data.port), pair_int_hash(key.hostname.hash(), key.port)); - } -}; - -namespace RequestServer::ConnectionCache { - -struct InferredServerProperties { - size_t requests_served_per_connection { NumericLimits::max() }; -}; - -extern Threading::RWLockProtected>>>>> g_tcp_connection_cache; -extern Threading::RWLockProtected>>>>> g_tls_connection_cache; -extern Threading::RWLockProtected> g_inferred_server_properties; - -void request_did_finish(URL::URL const&, Core::Socket const*); -void dump_jobs(); - -constexpr static size_t MaxConcurrentConnectionsPerURL = 4; -constexpr static size_t ConnectionKeepAliveTimeMilliseconds = 20'000; -constexpr static size_t ConnectionCacheQueueHighWatermark = 4; - -template -ErrorOr recreate_socket_if_needed(T& connection, URL::URL const& url) -{ - using SocketType = typename T::SocketType; - using SocketStorageType = typename T::StorageType; - - if (!connection.socket || !connection.socket->is_open() || connection.socket->is_eof()) { - connection.socket = nullptr; - // Create another socket for the connection. - auto set_socket = [&](NonnullOwnPtr&& socket) -> ErrorOr { - connection.socket = TRY(Core::BufferedSocket::create(move(socket))); - return {}; - }; - - if constexpr (IsSame) { - TLS::Options options; - options.set_alert_handler([&connection](TLS::AlertDescription alert) { - Core::NetworkJob::Error reason; - if (alert == TLS::AlertDescription::HANDSHAKE_FAILURE) - reason = Core::NetworkJob::Error::ProtocolFailed; - else if (alert == TLS::AlertDescription::DECRYPT_ERROR) - reason = Core::NetworkJob::Error::ConnectionFailed; - else - reason = Core::NetworkJob::Error::TransmissionFailed; - - if (connection.job_data->fail) - connection.job_data->fail(reason); - }); - options.set_certificate_provider([&connection]() -> Vector { - if (connection.job_data->provide_client_certificates) - return connection.job_data->provide_client_certificates(); - return {}; - }); - TRY(set_socket(TRY((connection.proxy.template tunnel(url, move(options)))))); - } else { - TRY(set_socket(TRY((connection.proxy.template tunnel(url))))); - } - dbgln_if(REQUESTSERVER_DEBUG, "Creating a new socket for {} -> {}", url, connection.socket.ptr()); - } - return {}; -} - -extern size_t hits; -extern size_t misses; - -template -void start_connection(const URL::URL& url, auto job, auto& sockets_for_url, size_t index, AK::Duration, Cache&); - -void ensure_connection(auto& cache, const URL::URL& url, auto job, Core::ProxyData proxy_data = {}) -{ - using CacheEntryType = RemoveCVReference::ProtectedType>().begin()->value)>; - - auto hostname = url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string(); - auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(hostname); }); - - auto& sockets_for_url = *cache.with_write_locked([&](auto& map) -> NonnullOwnPtr& { - return map.ensure({ move(hostname), url.port_or_default(), proxy_data }, [] { return make(); }); - }); - - // Find the connection with an empty queue; if none exist, we'll find the least backed-up connection later. - // Note that servers that are known to serve a single request per connection (e.g. HTTP/1.0) usually have - // issues with concurrent connections, so we'll only allow one connection per URL in that case to avoid issues. - // This is a bit too aggressive, but there's no way to know if the server can handle concurrent connections - // without trying it out first, and that's not worth the effort as HTTP/1.0 is a legacy protocol anyway. - auto it = sockets_for_url.find_if([&](auto const& connection) { - return properties.requests_served_per_connection < 2 - || connection->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }) <= ConnectionCacheQueueHighWatermark; - }); - auto did_add_new_connection = false; - auto failed_to_find_a_socket = it.is_end(); - - Proxy proxy { proxy_data }; - size_t index; - - auto timer = Core::ElapsedTimer::start_new(); - if (failed_to_find_a_socket && sockets_for_url.size() < MaxConcurrentConnectionsPerURL) { - using ConnectionType = RemoveCVReference().at(0))>; - auto& connection = cache.with_write_locked([&](auto&) -> ConnectionType& { - index = sockets_for_url.size(); - sockets_for_url.append(AK::make( - nullptr, - typename ConnectionType::QueueType {}, - Core::Timer::create_single_shot(ConnectionKeepAliveTimeMilliseconds, nullptr), - true)); - auto& connection = sockets_for_url.last(); - connection->proxy = move(proxy); - return *connection; - }); - ScopeGuard start_guard = [&] { - connection.is_being_started = false; - }; - dbgln_if(REQUESTSERVER_DEBUG, "I will start a connection ({}) for URL {}", &connection, url); - - auto connection_result = proxy.tunnel(url); - misses++; - if (connection_result.is_error()) { - dbgln("ConnectionCache: Connection to {} failed: {}", url, connection_result.error()); - Core::deferred_invoke([job] { - job->fail(Core::NetworkJob::Error::ConnectionFailed); - }); - return; - } - auto socket_result = Core::BufferedSocket::create(connection_result.release_value()); - if (socket_result.is_error()) { - dbgln("ConnectionCache: Failed to make a buffered socket for {}: {}", url, socket_result.error()); - Core::deferred_invoke([job] { - job->fail(Core::NetworkJob::Error::ConnectionFailed); - }); - return; - } - did_add_new_connection = true; - connection.socket = socket_result.release_value(); - } - - auto elapsed = AK::Duration::from_milliseconds(timer.elapsed_milliseconds()); - - if (failed_to_find_a_socket) { - if (!did_add_new_connection) { - // Find the least backed-up connection (based on how many entries are in their request queue). - index = 0; - auto min_queue_size = (size_t)-1; - for (auto it = sockets_for_url.begin(); it != sockets_for_url.end(); ++it) { - if (auto queue_size = (*it)->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }); min_queue_size > queue_size) { - index = it.index(); - min_queue_size = queue_size; - } - } - } - } else { - index = it.index(); - hits++; - } - - dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: Hits: {}, Misses: {}", RequestServer::ConnectionCache::hits, RequestServer::ConnectionCache::misses); - start_connection(url, job, sockets_for_url, index, elapsed, cache); -} - -template -void start_connection(URL::URL const& url, auto job, auto& sockets_for_url, size_t index, AK::Duration setup_time, Cache& cache) -{ - if (sockets_for_url.is_empty()) { - Core::deferred_invoke([job] { - job->fail(Core::NetworkJob::Error::ConnectionFailed); - }); - return; - } - - auto& connection = *sockets_for_url[index]; - if (connection.is_being_started) { - // Someone else is creating the connection, queue the job and let them handle it. - dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr()); - auto size = connection.request_queue.with_write_locked([&](auto& queue) { - queue.append(JobData::create(job, url)); - return queue.size(); - }); - connection.max_queue_length = max(connection.max_queue_length, size); - return; - } - - if (!connection.has_started) { - connection.has_started = true; - Core::deferred_invoke([&connection, &cache, url, job, setup_time] { - (void)setup_time; - auto job_data = JobData::create(job, url); - if constexpr (REQUESTSERVER_DEBUG) { - job_data.timing_info.waiting_in_queue = AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds()); - job_data.timing_info.timer.start(); - } - if (auto result = recreate_socket_if_needed(connection, url); result.is_error()) { - dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: request failed to start, failed to make a socket: {}", result.error()); - if constexpr (REQUESTSERVER_DEBUG) { - job_data.timing_info.starting_connection += AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds()) + setup_time; - job_data.timing_info.timer.start(); - } - Core::deferred_invoke([job] { - job->fail(Core::NetworkJob::Error::ConnectionFailed); - }); - } else { - cache.with_write_locked([&](auto&) { - dbgln_if(REQUESTSERVER_DEBUG, "Immediately start request for url {} in {} - {}", url, &connection, connection.socket.ptr()); - connection.job_data = move(job_data); - if constexpr (REQUESTSERVER_DEBUG) { - connection.job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds()) + setup_time; - connection.job_data->timing_info.timer.start(); - } - connection.removal_timer->stop(); - connection.timer.start(); - connection.current_url = url; - connection.socket->set_notifications_enabled(true); - connection.job_data->start(*connection.socket); - }); - } - }); - } else { - dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr()); - auto size = connection.request_queue.with_write_locked([&](auto& queue) { - queue.append(JobData::create(job, url)); - return queue.size(); - }); - connection.max_queue_length = max(connection.max_queue_length, size); - } -} -} diff --git a/Userland/Services/RequestServer/ConnectionFromClient.cpp b/Userland/Services/RequestServer/ConnectionFromClient.cpp index 368a3b4746d..95b6de61e87 100644 --- a/Userland/Services/RequestServer/ConnectionFromClient.cpp +++ b/Userland/Services/RequestServer/ConnectionFromClient.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/Userland/Services/RequestServer/HttpCommon.h b/Userland/Services/RequestServer/HttpCommon.h deleted file mode 100644 index 3cff2308971..00000000000 --- a/Userland/Services/RequestServer/HttpCommon.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2018-2021, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer::Detail { - -template -void init(TSelf* self, TJob job) -{ - job->on_headers_received = [self](auto& headers, auto response_code) { - if (response_code.has_value()) - self->set_status_code(response_code.value()); - self->set_response_headers(headers); - }; - - job->on_finish = [self](bool success) { - Core::deferred_invoke([url = self->job().url(), socket = self->job().socket()] { - ConnectionCache::request_did_finish(url, socket); - }); - if (auto* response = self->job().response()) { - self->set_status_code(response->code()); - self->set_response_headers(response->headers()); - self->set_downloaded_size(response->downloaded_size()); - } - - // if we didn't know the total size, pretend that the request finished successfully - // and set the total size to the downloaded size - if (!self->total_size().has_value()) - self->did_progress(self->downloaded_size(), self->downloaded_size()); - - self->did_finish(success); - }; - job->on_progress = [self](Optional total, u64 current) { - self->did_progress(total, current); - }; - if constexpr (requires { job->on_certificate_requested; }) { - job->on_certificate_requested = [job, self] { - self->did_request_certificates(); - Core::EventLoop::current().spin_until([&] { - return job->received_client_certificates(); - }); - return job->take_client_certificates(); - }; - } -} - -template -OwnPtr start_request(TBadgedProtocol&& protocol, i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, TPipeResult&& pipe_result, Core::ProxyData proxy_data = {}) -{ - using TJob = typename TBadgedProtocol::Type::JobType; - using TRequest = typename TBadgedProtocol::Type::RequestType; - - if (pipe_result.is_error()) { - return {}; - } - - HTTP::HttpRequest request; - if (method.equals_ignoring_ascii_case("post"sv)) - request.set_method(HTTP::HttpRequest::Method::POST); - else if (method.equals_ignoring_ascii_case("head"sv)) - request.set_method(HTTP::HttpRequest::Method::HEAD); - else if (method.equals_ignoring_ascii_case("delete"sv)) - request.set_method(HTTP::HttpRequest::Method::DELETE); - else if (method.equals_ignoring_ascii_case("patch"sv)) - request.set_method(HTTP::HttpRequest::Method::PATCH); - else if (method.equals_ignoring_ascii_case("options"sv)) - request.set_method(HTTP::HttpRequest::Method::OPTIONS); - else if (method.equals_ignoring_ascii_case("trace"sv)) - request.set_method(HTTP::HttpRequest::Method::TRACE); - else if (method.equals_ignoring_ascii_case("connect"sv)) - request.set_method(HTTP::HttpRequest::Method::CONNECT); - else if (method.equals_ignoring_ascii_case("put"sv)) - request.set_method(HTTP::HttpRequest::Method::PUT); - else - request.set_method(HTTP::HttpRequest::Method::GET); - request.set_url(url); - request.set_headers(headers); - - auto allocated_body_result = ByteBuffer::copy(body); - if (allocated_body_result.is_error()) - return {}; - request.set_body(allocated_body_result.release_value()); - - auto output_stream = MUST(Core::File::adopt_fd(pipe_result.value().write_fd, Core::File::OpenMode::Write)); - auto job = TJob::construct(move(request), *output_stream); - auto protocol_request = TRequest::create_with_job(forward(protocol), client, static_cast(*job), move(output_stream), request_id); - protocol_request->set_request_fd(pipe_result.value().read_fd); - - Core::deferred_invoke([=, job = job->template make_weak_ptr()] { - if constexpr (IsSame) - ConnectionCache::ensure_connection(ConnectionCache::g_tls_connection_cache, url, job, proxy_data); - else - ConnectionCache::ensure_connection(ConnectionCache::g_tcp_connection_cache, url, job, proxy_data); - }); - - return protocol_request; -} - -} diff --git a/Userland/Services/RequestServer/HttpProtocol.cpp b/Userland/Services/RequestServer/HttpProtocol.cpp deleted file mode 100644 index a4fcbd0bd98..00000000000 --- a/Userland/Services/RequestServer/HttpProtocol.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer { - -HttpProtocol::HttpProtocol() - : Protocol("http") -{ -} - -OwnPtr HttpProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data) -{ - return Detail::start_request(Badge {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data); -} - -void HttpProtocol::install() -{ - Protocol::install(adopt_own(*new HttpProtocol())); -} - -} diff --git a/Userland/Services/RequestServer/HttpProtocol.h b/Userland/Services/RequestServer/HttpProtocol.h deleted file mode 100644 index 500773dc278..00000000000 --- a/Userland/Services/RequestServer/HttpProtocol.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer { - -class HttpProtocol final : public Protocol { -public: - using JobType = HTTP::Job; - using RequestType = HttpRequest; - - ~HttpProtocol() override = default; - - static void install(); - -private: - HttpProtocol(); - - virtual OwnPtr start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override; -}; - -} diff --git a/Userland/Services/RequestServer/HttpRequest.cpp b/Userland/Services/RequestServer/HttpRequest.cpp deleted file mode 100644 index 28cae8c3a04..00000000000 --- a/Userland/Services/RequestServer/HttpRequest.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include - -namespace RequestServer { - -HttpRequest::HttpRequest(ConnectionFromClient& client, NonnullRefPtr job, NonnullOwnPtr&& output_stream, i32 request_id) - : Request(client, move(output_stream), request_id) - , m_job(job) -{ - Detail::init(this, job); -} - -HttpRequest::~HttpRequest() -{ - m_job->on_finish = nullptr; - m_job->on_progress = nullptr; - m_job->cancel(); -} - -NonnullOwnPtr HttpRequest::create_with_job(Badge&&, ConnectionFromClient& client, NonnullRefPtr job, NonnullOwnPtr&& output_stream, i32 request_id) -{ - return adopt_own(*new HttpRequest(client, move(job), move(output_stream), request_id)); -} - -} diff --git a/Userland/Services/RequestServer/HttpRequest.h b/Userland/Services/RequestServer/HttpRequest.h deleted file mode 100644 index ea18b599c53..00000000000 --- a/Userland/Services/RequestServer/HttpRequest.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace RequestServer { - -class HttpRequest final : public Request { -public: - virtual ~HttpRequest() override; - static NonnullOwnPtr create_with_job(Badge&&, ConnectionFromClient&, NonnullRefPtr, NonnullOwnPtr&&, i32); - - HTTP::Job& job() { return m_job; } - HTTP::Job const& job() const { return m_job; } - - virtual URL::URL url() const override { return m_job->url(); } - virtual void cancel() override { m_job->cancel(); } - -private: - explicit HttpRequest(ConnectionFromClient&, NonnullRefPtr, NonnullOwnPtr&&, i32); - - NonnullRefPtr m_job; -}; - -} diff --git a/Userland/Services/RequestServer/HttpsProtocol.cpp b/Userland/Services/RequestServer/HttpsProtocol.cpp deleted file mode 100644 index fff80e9f8f1..00000000000 --- a/Userland/Services/RequestServer/HttpsProtocol.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2018-2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer { - -HttpsProtocol::HttpsProtocol() - : Protocol("https") -{ -} - -OwnPtr HttpsProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data) -{ - return Detail::start_request(Badge {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data); -} - -void HttpsProtocol::install() -{ - Protocol::install(adopt_own(*new HttpsProtocol())); -} - -} diff --git a/Userland/Services/RequestServer/HttpsProtocol.h b/Userland/Services/RequestServer/HttpsProtocol.h deleted file mode 100644 index 3bf1df401bd..00000000000 --- a/Userland/Services/RequestServer/HttpsProtocol.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2018-2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace RequestServer { - -class HttpsProtocol final : public Protocol { -public: - using JobType = HTTP::HttpsJob; - using RequestType = HttpsRequest; - - ~HttpsProtocol() override = default; - - static void install(); - -private: - HttpsProtocol(); - - virtual OwnPtr start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override; -}; - -} diff --git a/Userland/Services/RequestServer/HttpsRequest.cpp b/Userland/Services/RequestServer/HttpsRequest.cpp deleted file mode 100644 index fd211065cc9..00000000000 --- a/Userland/Services/RequestServer/HttpsRequest.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include - -namespace RequestServer { - -HttpsRequest::HttpsRequest(ConnectionFromClient& client, NonnullRefPtr job, NonnullOwnPtr&& output_stream, i32 request_id) - : Request(client, move(output_stream), request_id) - , m_job(job) -{ - Detail::init(this, job); -} - -void HttpsRequest::set_certificate(ByteString certificate, ByteString key) -{ - m_job->set_certificate(move(certificate), move(key)); -} - -HttpsRequest::~HttpsRequest() -{ - m_job->on_finish = nullptr; - m_job->on_progress = nullptr; - m_job->cancel(); -} - -NonnullOwnPtr HttpsRequest::create_with_job(Badge&&, ConnectionFromClient& client, NonnullRefPtr job, NonnullOwnPtr&& output_stream, i32 request_id) -{ - return adopt_own(*new HttpsRequest(client, move(job), move(output_stream), request_id)); -} - -} diff --git a/Userland/Services/RequestServer/HttpsRequest.h b/Userland/Services/RequestServer/HttpsRequest.h deleted file mode 100644 index 1eb7e7097bb..00000000000 --- a/Userland/Services/RequestServer/HttpsRequest.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include - -namespace RequestServer { - -class HttpsRequest final : public Request { -public: - virtual ~HttpsRequest() override; - static NonnullOwnPtr create_with_job(Badge&&, ConnectionFromClient&, NonnullRefPtr, NonnullOwnPtr&&, i32); - - HTTP::HttpsJob& job() { return m_job; } - HTTP::HttpsJob const& job() const { return m_job; } - - virtual URL::URL url() const override { return m_job->url(); } - virtual void cancel() override { m_job->cancel(); } - -private: - explicit HttpsRequest(ConnectionFromClient&, NonnullRefPtr, NonnullOwnPtr&&, i32); - - virtual void set_certificate(ByteString certificate, ByteString key) override; - - NonnullRefPtr m_job; -}; - -} diff --git a/Userland/Services/RequestServer/Protocol.cpp b/Userland/Services/RequestServer/Protocol.cpp deleted file mode 100644 index 1dce3e95398..00000000000 --- a/Userland/Services/RequestServer/Protocol.cpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include - -namespace RequestServer { - -static HashMap>& all_protocols() -{ - static HashMap> map; - return map; -} - -Protocol* Protocol::find_by_name(ByteString const& name) -{ - return all_protocols().get(name).map([](auto& p) -> Protocol* { return p; }).value_or(nullptr); -} - -Protocol::Protocol(ByteString const& name) - : m_name(name) -{ -} - -ErrorOr Protocol::get_pipe_for_request() -{ - auto fd_pair = TRY(Core::System::pipe2(O_NONBLOCK)); - return Pipe { fd_pair[0], fd_pair[1] }; -} - -void Protocol::install(NonnullOwnPtr protocol) -{ - auto name = protocol->name(); - all_protocols().set(move(name), move(protocol)); -} - -} diff --git a/Userland/Services/RequestServer/Protocol.h b/Userland/Services/RequestServer/Protocol.h deleted file mode 100644 index 4a00b69000c..00000000000 --- a/Userland/Services/RequestServer/Protocol.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace RequestServer { - -class Protocol { -public: - virtual ~Protocol() = default; - - ByteString const& name() const { return m_name; } - virtual OwnPtr start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) = 0; - - static Protocol* find_by_name(ByteString const&); - -protected: - explicit Protocol(ByteString const& name); - struct Pipe { - int read_fd { -1 }; - int write_fd { -1 }; - }; - static ErrorOr get_pipe_for_request(); - - static void install(NonnullOwnPtr); - -private: - ByteString m_name; -}; - -} diff --git a/Userland/Services/RequestServer/RequestServer.ipc b/Userland/Services/RequestServer/RequestServer.ipc index 0e588136087..cc5762995ce 100644 --- a/Userland/Services/RequestServer/RequestServer.ipc +++ b/Userland/Services/RequestServer/RequestServer.ipc @@ -1,6 +1,7 @@ +#include #include #include -#include +#include endpoint RequestServer {