mirrors/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-04-20 11:36:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

RequestServer: Remove now-unused HTTP related code

Browse source
This commit is contained in:
Andreas Kling 2024-09-19 07:54:24 +02:00 • committed by Andreas Kling
parent a1fca1a7f3
commit 504c80a202
Notes: github-actions[bot] 2024-09-22 04:44:07 +00:00
18 changed files with 20 additions and 1044 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Userland/Libraries/LibWebView/CMakeLists.txt
View file

@ -67,6 +67,4 @@ if (ENABLE_INSTALL_HEADERS)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${header}" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${subdirectory}")
endforeach()
install(FILES "${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/ConnectionCache.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/RequestServer")
endif()

8
Userland/Services/RequestServer/CMakeLists.txt
View file

@ -3,14 +3,8 @@ compile_ipc(RequestClient.ipc RequestClientEndpoint.h)
set(SOURCES
ConnectionFromClient.cpp
ConnectionCache.cpp
Request.cpp
HttpRequest.cpp
HttpProtocol.cpp
HttpsRequest.cpp
HttpsProtocol.cpp
main.cpp
Protocol.cpp
)
set(GENERATED_SOURCES
@ -19,4 +13,4 @@ set(GENERATED_SOURCES
)
serenity_bin(RequestServer)
target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibHTTP LibMain LibTLS LibWebSocket LibURL LibThreading)
target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibMain LibTLS LibWebSocket LibURL LibThreading)

16
Userland/Services/RequestServer/CacheLevel.h Normal file
View file

@ -0,0 +1,16 @@
/*
* Copyright (c) 2024, Andreas Kling <andreas@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
namespace RequestServer {
enum class CacheLevel {
ResolveOnly,
CreateConnection,
};
}

167
Userland/Services/RequestServer/ConnectionCache.cpp
View file

@ -1,167 +0,0 @@
/*
* Copyright (c) 2021-2022, Ali Mohammad Pur <mpfard@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "ConnectionCache.h"
#include <AK/Debug.h>
#include <AK/Find.h>
#include <LibCore/EventLoop.h>
namespace RequestServer::ConnectionCache {
Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<Core::TCPSocket, Core::Socket>>>>>> g_tcp_connection_cache {};
Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<TLS::TLSv12>>>>>> g_tls_connection_cache {};
Threading::RWLockProtected<HashMap<ByteString, InferredServerProperties>> g_inferred_server_properties;
void request_did_finish(URL::URL const& url, Core::Socket const* socket)
{
if (!socket) {
dbgln("Request with a null socket finished for URL {}", url);
return;
}
dbgln_if(REQUESTSERVER_DEBUG, "Request for {} finished", url);
ConnectionKey partial_key { url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string(), url.port_or_default() };
auto fire_off_next_job = [&](auto& cache) {
using CacheType = typename RemoveCVReference<decltype(cache)>::ProtectedType;
auto [it, end] = cache.with_read_locked([&](auto const& cache) {
struct Result {
decltype(cache.begin()) it;
decltype(cache.end()) end;
};
return Result {
find_if(cache.begin(), cache.end(), [&](auto& connection) {
return connection.key.hostname == partial_key.hostname && connection.key.port == partial_key.port;
}),
cache.end(),
};
});
if (it == end) {
dbgln("Request for URL {} finished, but we don't own that!", url);
return;
}
auto connection_it = it->value->find_if([&](auto& connection) { return connection->socket == socket; });
if (connection_it.is_end()) {
dbgln("Request for URL {} finished, but we don't have a socket for that!", url);
return;
}
auto& connection = *connection_it;
if constexpr (REQUESTSERVER_DEBUG) {
connection->job_data->timing_info.performing_request = AK::Duration::from_milliseconds(connection->job_data->timing_info.timer.elapsed_milliseconds());
connection->job_data->timing_info.timer.start();
}
auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(partial_key.hostname); });
if (!connection->socket->is_open())
properties.requests_served_per_connection = min(properties.requests_served_per_connection, connection->max_queue_length + 1);
if (connection->request_queue.with_read_locked([](auto const& queue) { return queue.is_empty(); })) {
// Immediately mark the connection as finished, as new jobs will never be run if they are queued
// before the deferred_invoke() below runs otherwise.
connection->has_started = false;
connection->socket->set_notifications_enabled(false);
Core::deferred_invoke([&connection, &cache_entry = *it->value, key = it->key, &cache] {
if (connection->has_started)
return;
connection->current_url = {};
connection->job_data = {};
connection->removal_timer->on_timeout = [ptr = connection.ptr(), &cache_entry, key = move(key), &cache]() mutable {
Core::deferred_invoke([&, key = move(key), ptr] {
if (ptr->has_started)
return;
dbgln_if(REQUESTSERVER_DEBUG, "Removing no-longer-used connection {} (socket {})", ptr, ptr->socket.ptr());
cache.with_write_locked([&](CacheType& cache) {
auto did_remove = cache_entry.remove_first_matching([&](auto& entry) { return entry == ptr; });
VERIFY(did_remove);
if (cache_entry.is_empty())
cache.remove(key);
});
});
};
connection->removal_timer->start();
});
} else {
auto timer = Core::ElapsedTimer::start_new();
if (auto result = recreate_socket_if_needed(*connection, url); result.is_error()) {
if constexpr (REQUESTSERVER_DEBUG) {
connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
}
cache.with_read_locked([&](auto&) {
dbgln("ConnectionCache request finish handler, reconnection failed with {}", result.error());
connection->job_data->fail(Core::NetworkJob::Error::ConnectionFailed);
});
return;
}
if constexpr (REQUESTSERVER_DEBUG) {
connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
}
connection->has_started = true;
Core::deferred_invoke([&connection = *connection, url, &cache] {
cache.with_read_locked([&](auto&) {
dbgln_if(REQUESTSERVER_DEBUG, "Running next job in queue for connection {}", &connection);
connection.timer.start();
connection.current_url = url;
connection.job_data = connection.request_queue.with_write_locked([](auto& queue) { return queue.take_first(); });
if constexpr (REQUESTSERVER_DEBUG) {
connection.job_data->timing_info.waiting_in_queue = AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds() - connection.job_data->timing_info.performing_request.to_milliseconds());
connection.job_data->timing_info.timer.start();
}
connection.socket->set_notifications_enabled(true);
connection.job_data->start(*connection.socket);
});
});
}
};
if (is<Core::BufferedSocket<TLS::TLSv12>>(socket))
fire_off_next_job(g_tls_connection_cache);
else if (is<Core::BufferedSocket<Core::Socket>>(socket))
fire_off_next_job(g_tcp_connection_cache);
else
dbgln("Unknown socket {} finished for URL {}", socket, url);
}
void dump_jobs()
{
g_tls_connection_cache.with_read_locked([](auto& cache) {
dbgln("=========== TLS Connection Cache ==========");
for (auto& connection : cache) {
dbgln(" - {}:{}", connection.key.hostname, connection.key.port);
for (auto& entry : *connection.value) {
dbgln(" - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr());
dbgln(" Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0);
dbgln(" Request Queue:");
entry->request_queue.for_each_locked([](auto const& job) {
dbgln(" - {}", &job);
});
}
}
});
g_tcp_connection_cache.with_read_locked([](auto& cache) {
dbgln("=========== TCP Connection Cache ==========");
for (auto& connection : cache) {
dbgln(" - {}:{}", connection.key.hostname, connection.key.port);
for (auto& entry : *connection.value) {
dbgln(" - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr());
dbgln(" Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0);
dbgln(" Request Queue:");
entry->request_queue.for_each_locked([](auto const& job) {
dbgln(" - {}", &job);
});
}
}
});
}
size_t hits;
size_t misses;
}

386
Userland/Services/RequestServer/ConnectionCache.h
View file

@ -1,386 +0,0 @@
/*
* Copyright (c) 2021-2022, Ali Mohammad Pur <mpfard@serenityos.org>
* Copyright (c) 2022, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Debug.h>
#include <AK/HashMap.h>
#include <AK/Vector.h>
#include <LibCore/ElapsedTimer.h>
#include <LibCore/EventLoop.h>
#include <LibCore/NetworkJob.h>
#include <LibCore/SOCKSProxyClient.h>
#include <LibCore/Timer.h>
#include <LibTLS/TLSv12.h>
#include <LibThreading/RWLockProtected.h>
#include <LibURL/URL.h>
namespace RequestServer {
enum class CacheLevel {
ResolveOnly,
CreateConnection,
};
}
namespace RequestServer::ConnectionCache {
struct Proxy {
Core::ProxyData data;
OwnPtr<Core::SOCKSProxyClient> proxy_client_storage {};
template<typename SocketType, typename StorageType, typename... Args>
ErrorOr<NonnullOwnPtr<StorageType>> tunnel(URL::URL const& url, Args&&... args)
{
if (data.type == Core::ProxyData::Direct) {
return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), url.port_or_default(), forward<Args>(args)...));
}
if (data.type == Core::ProxyData::SOCKS5) {
if constexpr (requires { SocketType::connect(declval<ByteString>(), *proxy_client_storage, forward<Args>(args)...); }) {
proxy_client_storage = TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default()));
return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), *proxy_client_storage, forward<Args>(args)...));
} else if constexpr (IsSame<SocketType, Core::TCPSocket>) {
return TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default()));
} else {
return Error::from_string_literal("SOCKS5 not supported for this socket type");
}
}
VERIFY_NOT_REACHED();
}
};
struct JobData {
Function<void(Core::BufferedSocketBase&)> start {};
Function<void(Core::NetworkJob::Error)> fail {};
Function<Vector<TLS::Certificate>()> provide_client_certificates {};
struct TimingInfo {
#if REQUESTSERVER_DEBUG
bool valid { true };
Core::ElapsedTimer timer {};
URL::URL url {};
AK::Duration waiting_in_queue {};
AK::Duration starting_connection {};
AK::Duration performing_request {};
#endif
} timing_info {};
JobData(Function<void(Core::BufferedSocketBase&)> start, Function<void(Core::NetworkJob::Error)> fail, Function<Vector<TLS::Certificate>()> provide_client_certificates, TimingInfo timing_info)
: start(move(start))
, fail(move(fail))
, provide_client_certificates(move(provide_client_certificates))
, timing_info(move(timing_info))
{
}
JobData(JobData&& other)
: start(move(other.start))
, fail(move(other.fail))
, provide_client_certificates(move(other.provide_client_certificates))
, timing_info(move(other.timing_info))
{
#if REQUESTSERVER_DEBUG
other.timing_info.valid = false;
#endif
}
#if REQUESTSERVER_DEBUG
~JobData()
{
if (timing_info.valid) {
dbgln("JobData for {} timings:", timing_info.url);
dbgln(" - Waiting in queue: {}ms", timing_info.waiting_in_queue.to_milliseconds());
dbgln(" - Starting connection: {}ms", timing_info.starting_connection.to_milliseconds());
dbgln(" - Performing request: {}ms", timing_info.performing_request.to_milliseconds());
}
}
#endif
static JobData create(auto job, [[maybe_unused]] URL::URL url)
{
return JobData {
[job](auto& socket) { job->start(socket); },
[job](auto error) { job->fail(error); },
[job] {
if constexpr (requires { job->on_certificate_requested; }) {
if (job->on_certificate_requested)
return job->on_certificate_requested();
} else {
// "use" `job`, otherwise clang gets sad.
(void)job;
}
return Vector<TLS::Certificate> {};
},
{
#if REQUESTSERVER_DEBUG
.timer = Core::ElapsedTimer::start_new(Core::TimerType::Precise),
.url = move(url),
.waiting_in_queue = {},
.starting_connection = {},
.performing_request = {},
#endif
},
};
}
};
template<typename Socket, typename SocketStorageType = Socket>
struct Connection {
using QueueType = Vector<JobData>;
using SocketType = Socket;
using StorageType = SocketStorageType;
OwnPtr<Core::BufferedSocket<SocketStorageType>> socket;
Threading::RWLockProtected<QueueType> request_queue;
NonnullRefPtr<Core::Timer> removal_timer;
Atomic<bool> is_being_started { false };
bool has_started { false };
URL::URL current_url {};
Core::ElapsedTimer timer {};
Optional<JobData> job_data {};
Proxy proxy {};
size_t max_queue_length { 0 };
};
struct ConnectionKey {
ByteString hostname;
u16 port { 0 };
Core::ProxyData proxy_data {};
bool operator==(ConnectionKey const&) const = default;
};
};
template<>
struct AK::Traits<RequestServer::ConnectionCache::ConnectionKey> : public AK::DefaultTraits<RequestServer::ConnectionCache::ConnectionKey> {
static u32 hash(RequestServer::ConnectionCache::ConnectionKey const& key)
{
return pair_int_hash(pair_int_hash(key.proxy_data.host_ipv4, key.proxy_data.port), pair_int_hash(key.hostname.hash(), key.port));
}
};
namespace RequestServer::ConnectionCache {
struct InferredServerProperties {
size_t requests_served_per_connection { NumericLimits<size_t>::max() };
};
extern Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<Core::TCPSocket, Core::Socket>>>>>> g_tcp_connection_cache;
extern Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<TLS::TLSv12>>>>>> g_tls_connection_cache;
extern Threading::RWLockProtected<HashMap<ByteString, InferredServerProperties>> g_inferred_server_properties;
void request_did_finish(URL::URL const&, Core::Socket const*);
void dump_jobs();
constexpr static size_t MaxConcurrentConnectionsPerURL = 4;
constexpr static size_t ConnectionKeepAliveTimeMilliseconds = 20'000;
constexpr static size_t ConnectionCacheQueueHighWatermark = 4;
template<typename T>
ErrorOr<void> recreate_socket_if_needed(T& connection, URL::URL const& url)
{
using SocketType = typename T::SocketType;
using SocketStorageType = typename T::StorageType;
if (!connection.socket || !connection.socket->is_open() || connection.socket->is_eof()) {
connection.socket = nullptr;
// Create another socket for the connection.
auto set_socket = [&](NonnullOwnPtr<SocketStorageType>&& socket) -> ErrorOr<void> {
connection.socket = TRY(Core::BufferedSocket<SocketStorageType>::create(move(socket)));
return {};
};
if constexpr (IsSame<TLS::TLSv12, SocketType>) {
TLS::Options options;
options.set_alert_handler([&connection](TLS::AlertDescription alert) {
Core::NetworkJob::Error reason;
if (alert == TLS::AlertDescription::HANDSHAKE_FAILURE)
reason = Core::NetworkJob::Error::ProtocolFailed;
else if (alert == TLS::AlertDescription::DECRYPT_ERROR)
reason = Core::NetworkJob::Error::ConnectionFailed;
else
reason = Core::NetworkJob::Error::TransmissionFailed;
if (connection.job_data->fail)
connection.job_data->fail(reason);
});
options.set_certificate_provider([&connection]() -> Vector<TLS::Certificate> {
if (connection.job_data->provide_client_certificates)
return connection.job_data->provide_client_certificates();
return {};
});
TRY(set_socket(TRY((connection.proxy.template tunnel<SocketType, SocketStorageType>(url, move(options))))));
} else {
TRY(set_socket(TRY((connection.proxy.template tunnel<SocketType, SocketStorageType>(url)))));
}
dbgln_if(REQUESTSERVER_DEBUG, "Creating a new socket for {} -> {}", url, connection.socket.ptr());
}
return {};
}
extern size_t hits;
extern size_t misses;
template<typename Cache>
void start_connection(const URL::URL& url, auto job, auto& sockets_for_url, size_t index, AK::Duration, Cache&);
void ensure_connection(auto& cache, const URL::URL& url, auto job, Core::ProxyData proxy_data = {})
{
using CacheEntryType = RemoveCVReference<decltype(*declval<typename RemoveCVReference<decltype(cache)>::ProtectedType>().begin()->value)>;
auto hostname = url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string();
auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(hostname); });
auto& sockets_for_url = *cache.with_write_locked([&](auto& map) -> NonnullOwnPtr<CacheEntryType>& {
return map.ensure({ move(hostname), url.port_or_default(), proxy_data }, [] { return make<CacheEntryType>(); });
});
// Find the connection with an empty queue; if none exist, we'll find the least backed-up connection later.
// Note that servers that are known to serve a single request per connection (e.g. HTTP/1.0) usually have
// issues with concurrent connections, so we'll only allow one connection per URL in that case to avoid issues.
// This is a bit too aggressive, but there's no way to know if the server can handle concurrent connections
// without trying it out first, and that's not worth the effort as HTTP/1.0 is a legacy protocol anyway.
auto it = sockets_for_url.find_if([&](auto const& connection) {
return properties.requests_served_per_connection < 2
|| connection->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }) <= ConnectionCacheQueueHighWatermark;
});
auto did_add_new_connection = false;
auto failed_to_find_a_socket = it.is_end();
Proxy proxy { proxy_data };
size_t index;
auto timer = Core::ElapsedTimer::start_new();
if (failed_to_find_a_socket && sockets_for_url.size() < MaxConcurrentConnectionsPerURL) {
using ConnectionType = RemoveCVReference<decltype(*AK::Detail::declval<CacheEntryType>().at(0))>;
auto& connection = cache.with_write_locked([&](auto&) -> ConnectionType& {
index = sockets_for_url.size();
sockets_for_url.append(AK::make<ConnectionType>(
nullptr,
typename ConnectionType::QueueType {},
Core::Timer::create_single_shot(ConnectionKeepAliveTimeMilliseconds, nullptr),
true));
auto& connection = sockets_for_url.last();
connection->proxy = move(proxy);
return *connection;
});
ScopeGuard start_guard = [&] {
connection.is_being_started = false;
};
dbgln_if(REQUESTSERVER_DEBUG, "I will start a connection ({}) for URL {}", &connection, url);
auto connection_result = proxy.tunnel<typename ConnectionType::SocketType, typename ConnectionType::StorageType>(url);
misses++;
if (connection_result.is_error()) {
dbgln("ConnectionCache: Connection to {} failed: {}", url, connection_result.error());
Core::deferred_invoke([job] {
job->fail(Core::NetworkJob::Error::ConnectionFailed);
});
return;
}
auto socket_result = Core::BufferedSocket<typename ConnectionType::StorageType>::create(connection_result.release_value());
if (socket_result.is_error()) {
dbgln("ConnectionCache: Failed to make a buffered socket for {}: {}", url, socket_result.error());
Core::deferred_invoke([job] {
job->fail(Core::NetworkJob::Error::ConnectionFailed);
});
return;
}
did_add_new_connection = true;
connection.socket = socket_result.release_value();
}
auto elapsed = AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
if (failed_to_find_a_socket) {
if (!did_add_new_connection) {
// Find the least backed-up connection (based on how many entries are in their request queue).
index = 0;
auto min_queue_size = (size_t)-1;
for (auto it = sockets_for_url.begin(); it != sockets_for_url.end(); ++it) {
if (auto queue_size = (*it)->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }); min_queue_size > queue_size) {
index = it.index();
min_queue_size = queue_size;
}
}
}
} else {
index = it.index();
hits++;
}
dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: Hits: {}, Misses: {}", RequestServer::ConnectionCache::hits, RequestServer::ConnectionCache::misses);
start_connection(url, job, sockets_for_url, index, elapsed, cache);
}
template<typename Cache>
void start_connection(URL::URL const& url, auto job, auto& sockets_for_url, size_t index, AK::Duration setup_time, Cache& cache)
{
if (sockets_for_url.is_empty()) {
Core::deferred_invoke([job] {
job->fail(Core::NetworkJob::Error::ConnectionFailed);
});
return;
}
auto& connection = *sockets_for_url[index];
if (connection.is_being_started) {
// Someone else is creating the connection, queue the job and let them handle it.
dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr());
auto size = connection.request_queue.with_write_locked([&](auto& queue) {
queue.append(JobData::create(job, url));
return queue.size();
});
connection.max_queue_length = max(connection.max_queue_length, size);
return;
}
if (!connection.has_started) {
connection.has_started = true;
Core::deferred_invoke([&connection, &cache, url, job, setup_time] {
(void)setup_time;
auto job_data = JobData::create(job, url);
if constexpr (REQUESTSERVER_DEBUG) {
job_data.timing_info.waiting_in_queue = AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds());
job_data.timing_info.timer.start();
}
if (auto result = recreate_socket_if_needed(connection, url); result.is_error()) {
dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: request failed to start, failed to make a socket: {}", result.error());
if constexpr (REQUESTSERVER_DEBUG) {
job_data.timing_info.starting_connection += AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds()) + setup_time;
job_data.timing_info.timer.start();
}
Core::deferred_invoke([job] {
job->fail(Core::NetworkJob::Error::ConnectionFailed);
});
} else {
cache.with_write_locked([&](auto&) {
dbgln_if(REQUESTSERVER_DEBUG, "Immediately start request for url {} in {} - {}", url, &connection, connection.socket.ptr());
connection.job_data = move(job_data);
if constexpr (REQUESTSERVER_DEBUG) {
connection.job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds()) + setup_time;
connection.job_data->timing_info.timer.start();
}
connection.removal_timer->stop();
connection.timer.start();
connection.current_url = url;
connection.socket->set_notifications_enabled(true);
connection.job_data->start(*connection.socket);
});
}
});
} else {
dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr());
auto size = connection.request_queue.with_write_locked([&](auto& queue) {
queue.append(JobData::create(job, url));
return queue.size();
});
connection.max_queue_length = max(connection.max_queue_length, size);
}
}
}

1
Userland/Services/RequestServer/ConnectionFromClient.cpp
View file

@ -9,6 +9,7 @@
#include <AK/NonnullOwnPtr.h>
#include <AK/RefCounted.h>
#include <AK/Weakable.h>
#include <LibCore/EventLoop.h>
#include <LibCore/Proxy.h>
#include <LibCore/Socket.h>
#include <LibWebSocket/ConnectionInfo.h>

115
Userland/Services/RequestServer/HttpCommon.h
View file

@ -1,115 +0,0 @@
/*
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/ByteBuffer.h>
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/NonnullOwnPtr.h>
#include <AK/Optional.h>
#include <AK/OwnPtr.h>
#include <AK/Types.h>
#include <LibHTTP/HttpRequest.h>
#include <RequestServer/ConnectionCache.h>
#include <RequestServer/ConnectionFromClient.h>
#include <RequestServer/Request.h>
namespace RequestServer::Detail {
template<typename TSelf, typename TJob>
void init(TSelf* self, TJob job)
{
job->on_headers_received = [self](auto& headers, auto response_code) {
if (response_code.has_value())
self->set_status_code(response_code.value());
self->set_response_headers(headers);
};
job->on_finish = [self](bool success) {
Core::deferred_invoke([url = self->job().url(), socket = self->job().socket()] {
ConnectionCache::request_did_finish(url, socket);
});
if (auto* response = self->job().response()) {
self->set_status_code(response->code());
self->set_response_headers(response->headers());
self->set_downloaded_size(response->downloaded_size());
}
// if we didn't know the total size, pretend that the request finished successfully
// and set the total size to the downloaded size
if (!self->total_size().has_value())
self->did_progress(self->downloaded_size(), self->downloaded_size());
self->did_finish(success);
};
job->on_progress = [self](Optional<u64> total, u64 current) {
self->did_progress(total, current);
};
if constexpr (requires { job->on_certificate_requested; }) {
job->on_certificate_requested = [job, self] {
self->did_request_certificates();
Core::EventLoop::current().spin_until([&] {
return job->received_client_certificates();
});
return job->take_client_certificates();
};
}
}
template<typename TBadgedProtocol, typename TPipeResult>
OwnPtr<Request> start_request(TBadgedProtocol&& protocol, i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, TPipeResult&& pipe_result, Core::ProxyData proxy_data = {})
{
using TJob = typename TBadgedProtocol::Type::JobType;
using TRequest = typename TBadgedProtocol::Type::RequestType;
if (pipe_result.is_error()) {
return {};
}
HTTP::HttpRequest request;
if (method.equals_ignoring_ascii_case("post"sv))
request.set_method(HTTP::HttpRequest::Method::POST);
else if (method.equals_ignoring_ascii_case("head"sv))
request.set_method(HTTP::HttpRequest::Method::HEAD);
else if (method.equals_ignoring_ascii_case("delete"sv))
request.set_method(HTTP::HttpRequest::Method::DELETE);
else if (method.equals_ignoring_ascii_case("patch"sv))
request.set_method(HTTP::HttpRequest::Method::PATCH);
else if (method.equals_ignoring_ascii_case("options"sv))
request.set_method(HTTP::HttpRequest::Method::OPTIONS);
else if (method.equals_ignoring_ascii_case("trace"sv))
request.set_method(HTTP::HttpRequest::Method::TRACE);
else if (method.equals_ignoring_ascii_case("connect"sv))
request.set_method(HTTP::HttpRequest::Method::CONNECT);
else if (method.equals_ignoring_ascii_case("put"sv))
request.set_method(HTTP::HttpRequest::Method::PUT);
else
request.set_method(HTTP::HttpRequest::Method::GET);
request.set_url(url);
request.set_headers(headers);
auto allocated_body_result = ByteBuffer::copy(body);
if (allocated_body_result.is_error())
return {};
request.set_body(allocated_body_result.release_value());
auto output_stream = MUST(Core::File::adopt_fd(pipe_result.value().write_fd, Core::File::OpenMode::Write));
auto job = TJob::construct(move(request), *output_stream);
auto protocol_request = TRequest::create_with_job(forward<TBadgedProtocol>(protocol), client, static_cast<TJob&>(*job), move(output_stream), request_id);
protocol_request->set_request_fd(pipe_result.value().read_fd);
Core::deferred_invoke([=, job = job->template make_weak_ptr<TJob>()] {
if constexpr (IsSame<typename TBadgedProtocol::Type, HttpsProtocol>)
ConnectionCache::ensure_connection(ConnectionCache::g_tls_connection_cache, url, job, proxy_data);
else
ConnectionCache::ensure_connection(ConnectionCache::g_tcp_connection_cache, url, job, proxy_data);
});
return protocol_request;
}
}

35
Userland/Services/RequestServer/HttpProtocol.cpp
View file

@ -1,35 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Badge.h>
#include <AK/ByteBuffer.h>
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/OwnPtr.h>
#include <LibURL/URL.h>
#include <RequestServer/ConnectionFromClient.h>
#include <RequestServer/HttpCommon.h>
#include <RequestServer/HttpProtocol.h>
#include <RequestServer/Request.h>
namespace RequestServer {
HttpProtocol::HttpProtocol()
: Protocol("http")
{
}
OwnPtr<Request> HttpProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data)
{
return Detail::start_request(Badge<HttpProtocol> {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data);
}
void HttpProtocol::install()
{
Protocol::install(adopt_own(*new HttpProtocol()));
}
}

37
Userland/Services/RequestServer/HttpProtocol.h
View file

@ -1,37 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/ByteBuffer.h>
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/OwnPtr.h>
#include <LibHTTP/Job.h>
#include <LibURL/URL.h>
#include <RequestServer/ConnectionFromClient.h>
#include <RequestServer/HttpRequest.h>
#include <RequestServer/Protocol.h>
#include <RequestServer/Request.h>
namespace RequestServer {
class HttpProtocol final : public Protocol {
public:
using JobType = HTTP::Job;
using RequestType = HttpRequest;
~HttpProtocol() override = default;
static void install();
private:
HttpProtocol();
virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override;
};
}

33
Userland/Services/RequestServer/HttpRequest.cpp
View file

@ -1,33 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibHTTP/Job.h>
#include <RequestServer/HttpCommon.h>
#include <RequestServer/HttpProtocol.h>
#include <RequestServer/HttpRequest.h>
namespace RequestServer {
HttpRequest::HttpRequest(ConnectionFromClient& client, NonnullRefPtr<HTTP::Job> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
: Request(client, move(output_stream), request_id)
, m_job(job)
{
Detail::init(this, job);
}
HttpRequest::~HttpRequest()
{
m_job->on_finish = nullptr;
m_job->on_progress = nullptr;
m_job->cancel();
}
NonnullOwnPtr<HttpRequest> HttpRequest::create_with_job(Badge<HttpProtocol>&&, ConnectionFromClient& client, NonnullRefPtr<HTTP::Job> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
{
return adopt_own(*new HttpRequest(client, move(job), move(output_stream), request_id));
}
}

34
Userland/Services/RequestServer/HttpRequest.h
View file

@ -1,34 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Badge.h>
#include <AK/NonnullOwnPtr.h>
#include <LibCore/Forward.h>
#include <LibHTTP/Forward.h>
#include <RequestServer/Request.h>
namespace RequestServer {
class HttpRequest final : public Request {
public:
virtual ~HttpRequest() override;
static NonnullOwnPtr<HttpRequest> create_with_job(Badge<HttpProtocol>&&, ConnectionFromClient&, NonnullRefPtr<HTTP::Job>, NonnullOwnPtr<Core::File>&&, i32);
HTTP::Job& job() { return m_job; }
HTTP::Job const& job() const { return m_job; }
virtual URL::URL url() const override { return m_job->url(); }
virtual void cancel() override { m_job->cancel(); }
private:
explicit HttpRequest(ConnectionFromClient&, NonnullRefPtr<HTTP::Job>, NonnullOwnPtr<Core::File>&&, i32);
NonnullRefPtr<HTTP::Job> m_job;
};
}

35
Userland/Services/RequestServer/HttpsProtocol.cpp
View file

@ -1,35 +0,0 @@
/*
* Copyright (c) 2018-2020, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Badge.h>
#include <AK/ByteBuffer.h>
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/OwnPtr.h>
#include <LibURL/URL.h>
#include <RequestServer/ConnectionFromClient.h>
#include <RequestServer/HttpCommon.h>
#include <RequestServer/HttpsProtocol.h>
#include <RequestServer/Request.h>
namespace RequestServer {
HttpsProtocol::HttpsProtocol()
: Protocol("https")
{
}
OwnPtr<Request> HttpsProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data)
{
return Detail::start_request(Badge<HttpsProtocol> {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data);
}
void HttpsProtocol::install()
{
Protocol::install(adopt_own(*new HttpsProtocol()));
}
}

37
Userland/Services/RequestServer/HttpsProtocol.h
View file

@ -1,37 +0,0 @@
/*
* Copyright (c) 2018-2020, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/ByteBuffer.h>
#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/OwnPtr.h>
#include <LibHTTP/HttpsJob.h>
#include <LibURL/URL.h>
#include <RequestServer/ConnectionFromClient.h>
#include <RequestServer/HttpsRequest.h>
#include <RequestServer/Protocol.h>
#include <RequestServer/Request.h>
namespace RequestServer {
class HttpsProtocol final : public Protocol {
public:
using JobType = HTTP::HttpsJob;
using RequestType = HttpsRequest;
~HttpsProtocol() override = default;
static void install();
private:
HttpsProtocol();
virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override;
};
}

38
Userland/Services/RequestServer/HttpsRequest.cpp
View file

@ -1,38 +0,0 @@
/*
* Copyright (c) 2020, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibHTTP/HttpsJob.h>
#include <RequestServer/HttpCommon.h>
#include <RequestServer/HttpsProtocol.h>
#include <RequestServer/HttpsRequest.h>
namespace RequestServer {
HttpsRequest::HttpsRequest(ConnectionFromClient& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
: Request(client, move(output_stream), request_id)
, m_job(job)
{
Detail::init(this, job);
}
void HttpsRequest::set_certificate(ByteString certificate, ByteString key)
{
m_job->set_certificate(move(certificate), move(key));
}
HttpsRequest::~HttpsRequest()
{
m_job->on_finish = nullptr;
m_job->on_progress = nullptr;
m_job->cancel();
}
NonnullOwnPtr<HttpsRequest> HttpsRequest::create_with_job(Badge<HttpsProtocol>&&, ConnectionFromClient& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
{
return adopt_own(*new HttpsRequest(client, move(job), move(output_stream), request_id));
}
}

35
Userland/Services/RequestServer/HttpsRequest.h
View file

@ -1,35 +0,0 @@
/*
* Copyright (c) 2020, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Badge.h>
#include <LibCore/Forward.h>
#include <LibHTTP/HttpsJob.h>
#include <RequestServer/Request.h>
namespace RequestServer {
class HttpsRequest final : public Request {
public:
virtual ~HttpsRequest() override;
static NonnullOwnPtr<HttpsRequest> create_with_job(Badge<HttpsProtocol>&&, ConnectionFromClient&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<Core::File>&&, i32);
HTTP::HttpsJob& job() { return m_job; }
HTTP::HttpsJob const& job() const { return m_job; }
virtual URL::URL url() const override { return m_job->url(); }
virtual void cancel() override { m_job->cancel(); }
private:
explicit HttpsRequest(ConnectionFromClient&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<Core::File>&&, i32);
virtual void set_certificate(ByteString certificate, ByteString key) override;
NonnullRefPtr<HTTP::HttpsJob> m_job;
};
}

42
Userland/Services/RequestServer/Protocol.cpp
View file

@ -1,42 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/HashMap.h>
#include <AK/NonnullOwnPtr.h>
#include <LibCore/System.h>
#include <RequestServer/Protocol.h>
namespace RequestServer {
static HashMap<ByteString, NonnullOwnPtr<Protocol>>& all_protocols()
{
static HashMap<ByteString, NonnullOwnPtr<Protocol>> map;
return map;
}
Protocol* Protocol::find_by_name(ByteString const& name)
{
return all_protocols().get(name).map([](auto& p) -> Protocol* { return p; }).value_or(nullptr);
}
Protocol::Protocol(ByteString const& name)
: m_name(name)
{
}
ErrorOr<Protocol::Pipe> Protocol::get_pipe_for_request()
{
auto fd_pair = TRY(Core::System::pipe2(O_NONBLOCK));
return Pipe { fd_pair[0], fd_pair[1] };
}
void Protocol::install(NonnullOwnPtr<Protocol> protocol)
{
auto name = protocol->name();
all_protocols().set(move(name), move(protocol));
}
}

40
Userland/Services/RequestServer/Protocol.h
View file

@ -1,40 +0,0 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/RefPtr.h>
#include <LibCore/Proxy.h>
#include <LibHTTP/HeaderMap.h>
#include <LibURL/URL.h>
#include <RequestServer/Forward.h>
namespace RequestServer {
class Protocol {
public:
virtual ~Protocol() = default;
ByteString const& name() const { return m_name; }
virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) = 0;
static Protocol* find_by_name(ByteString const&);
protected:
explicit Protocol(ByteString const& name);
struct Pipe {
int read_fd { -1 };
int write_fd { -1 };
};
static ErrorOr<Pipe> get_pipe_for_request();
static void install(NonnullOwnPtr<Protocol>);
private:
ByteString m_name;
};
}

3
Userland/Services/RequestServer/RequestServer.ipc
View file

@ -1,6 +1,7 @@
#include <LibCore/Proxy.h>
#include <LibHTTP/HeaderMap.h>
#include <LibURL/URL.h>
#include <RequestServer/ConnectionCache.h>
#include <RequestServer/CacheLevel.h>
endpoint RequestServer
{