LibIPC: Port to Windows

The Linux IPC uses SCM_RIGHTS to transfer fds to another process
(see TransportSocket::transfer, which calls LocalSocket::send_message).
File descriptors are handled separately from regular data.

On Windows handles are embedded in regular data. They are duplicated
in the sender process.

Socket handles need special code both on sender side (because they
require using WSADuplicateSocket instead of DuplicateHandle, see
TransportSocketWindows::duplicate_handles) and on receiver side
(because they require WSASocket, see FileWindows.cpp).

TransportSocketWindows::ReadResult::fds vector is always empty, it is
kept the same as Linux version to avoid OS #ifdefs in Connection.h/.cpp
and Web::HTML::MessagePort::read_from_transport. Separate handling of
fds permeates all IPC code, it doesn't make sense to #ifdef out all this
code on Windows. In other words, the Linux code is more generic -
it handles both regular data and fds. On Windows, we need only the
regular data portion of it, and we just use that.

Duplicating handles on Windows requires pid of target (receiver)
process (see TransportSocketWindows::m_peer_pid). This pid is received
during special TransportSocketWindows initialization, which is performed
only on Windows. It is handled in a separate PR #3179.
Note: ChatGPT and [stackoverflow](https://stackoverflow.com/questions/25429887/getting-pid-of-peer-socket-on-windows) suggest using GetExtendedTcpTable/GetTcpTable2
to get peer pid, but this doesn't work because [MIB_TCPROW2::dwOwningPid](https://learn.microsoft.com/en-us/windows/win32/api/tcpmib/ns-tcpmib-mib_tcprow2)
is "The PID of the process that issued a context bind for this TCP
connection.", so for both ends it will return the pid of the process
that called socketpair.

Co-Authored-By: Andrew Kaster <andrew@ladybird.org>

Libraries/LibIPC/CMakeLists.txt

@@ -2,11 +2,18 @@ set(SOURCES
     Connection.cpp
     Decoder.cpp
     Encoder.cpp
-    Message.cpp
 )
 
 if (UNIX)
-    list(APPEND SOURCES TransportSocket.cpp)
+    list(APPEND SOURCES
+        File.cpp
+        Message.cpp
+        TransportSocket.cpp)
+else()
+    list(APPEND SOURCES
+        FileWindows.cpp
+        MessageWindows.cpp
+        TransportSocketWindows.cpp)
 endif()
 
 serenity_lib(LibIPC ipc)

Libraries/LibIPC/Connection.h

@@ -57,7 +57,7 @@ protected:
     RefPtr<Core::Timer> m_responsiveness_timer;
 
     Vector<NonnullOwnPtr<Message>> m_unprocessed_messages;
-    Queue<IPC::File> m_unprocessed_fds;
+    Queue<IPC::File> m_unprocessed_fds; // unused on Windows
     ByteBuffer m_unprocessed_bytes;
 
     u32 m_local_endpoint_magic { 0 };

Libraries/LibIPC/Decoder.cpp

@@ -14,7 +14,6 @@
 #include <LibIPC/Decoder.h>
 #include <LibIPC/File.h>
 #include <LibURL/URL.h>
-#include <fcntl.h>
 
 namespace IPC {
 
@@ -120,17 +119,6 @@ ErrorOr<URL::Host> decode(Decoder& decoder)
     return URL::Host { move(value) };
 }
 
-template<>
-ErrorOr<File> decode(Decoder& decoder)
-{
-    auto file = TRY(decoder.files().try_dequeue());
-    auto fd = file.fd();
-
-    auto fd_flags = TRY(Core::System::fcntl(fd, F_GETFD));
-    TRY(Core::System::fcntl(fd, F_SETFD, fd_flags | FD_CLOEXEC));
-    return file;
-}
-
 template<>
 ErrorOr<Empty> decode(Decoder&)
 {

Libraries/LibIPC/File.cpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2020, Sergey Bugaev <bugaevc@serenityos.org>
+ * Copyright (c) 2021, Andreas Kling <andreas@ladybird.org>
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibIPC/Decoder.h>
+#include <LibIPC/File.h>
+
+namespace IPC {
+
+// FIXME: IPC::Files transferred over the wire are always set O_CLOEXEC during decoding.
+//        Perhaps we should add an option to IPC::File to allow the receiver to decide whether to
+//        make it O_CLOEXEC or not. Or an attribute in the .ipc file?
+ErrorOr<void> File::clear_close_on_exec()
+{
+    auto fd_flags = TRY(Core::System::fcntl(m_fd, F_GETFD));
+    fd_flags &= ~FD_CLOEXEC;
+    TRY(Core::System::fcntl(m_fd, F_SETFD, fd_flags));
+    return {};
+}
+
+template<>
+ErrorOr<File> decode(Decoder& decoder)
+{
+    auto file = TRY(decoder.files().try_dequeue());
+    auto fd = file.fd();
+
+    auto fd_flags = TRY(Core::System::fcntl(fd, F_GETFD));
+    TRY(Core::System::fcntl(fd, F_SETFD, fd_flags | FD_CLOEXEC));
+    return file;
+}
+
+}

Libraries/LibIPC/File.h

@@ -63,16 +63,7 @@ public:
         return exchange(m_fd, -1);
     }
 
-    // FIXME: IPC::Files transferred over the wire are always set O_CLOEXEC during decoding.
-    //        Perhaps we should add an option to IPC::File to allow the receiver to decide whether to
-    //        make it O_CLOEXEC or not. Or an attribute in the .ipc file?
-    ErrorOr<void> clear_close_on_exec()
-    {
-        auto fd_flags = TRY(Core::System::fcntl(m_fd, F_GETFD));
-        fd_flags &= ~FD_CLOEXEC;
-        TRY(Core::System::fcntl(m_fd, F_SETFD, fd_flags));
-        return {};
-    }
+    ErrorOr<void> clear_close_on_exec();
 
 private:
     explicit File(int fd)

Libraries/LibIPC/FileWindows.cpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2025, stasoid <stasoid@yahoo.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibIPC/Decoder.h>
+#include <LibIPC/File.h>
+#include <LibIPC/HandleType.h>
+
+#include <AK/Windows.h>
+
+namespace IPC {
+
+ErrorOr<void> File::clear_close_on_exec()
+{
+    if (!SetHandleInformation(to_handle(m_fd), HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT))
+        return Error::from_windows_error();
+    return {};
+}
+
+template<>
+ErrorOr<File> decode(Decoder& decoder)
+{
+    auto handle_type = TRY(decoder.decode<HandleType>());
+    int handle = -1;
+    if (handle_type == HandleType::Generic) {
+        TRY(decoder.decode_into(handle));
+    } else if (handle_type == HandleType::Socket) {
+        WSAPROTOCOL_INFO pi = {};
+        TRY(decoder.decode_into({ reinterpret_cast<u8*>(&pi), sizeof(pi) }));
+        handle = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, &pi, 0, WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT);
+        if (handle == -1)
+            return Error::from_windows_error();
+    } else {
+        return Error::from_string_literal("Invalid handle type");
+    }
+    return File::adopt_fd(handle);
+}
+
+}

Libraries/LibIPC/HandleType.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) 2025, stasoid <stasoid@yahoo.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Types.h>
+
+namespace IPC {
+
+enum class HandleType : u8 {
+    Generic,
+    Socket
+};
+
+}

Libraries/LibIPC/Message.h

@@ -12,8 +12,8 @@
 #include <AK/RefPtr.h>
 #include <AK/Vector.h>
 #include <LibCore/Forward.h>
+#include <LibCore/System.h>
 #include <LibIPC/Transport.h>
-#include <unistd.h>
 
 namespace IPC {
 
@@ -27,7 +27,7 @@ public:
     ~AutoCloseFileDescriptor()
     {
         if (m_fd != -1)
-            close(m_fd);
+            (void)Core::System::close(m_fd);
     }
 
     int value() const { return m_fd; }
@@ -45,11 +45,14 @@ public:
 
     ErrorOr<void> append_file_descriptor(int fd);
 
-    ErrorOr<void> transfer_message(Transport& socket);
+    ErrorOr<void> transfer_message(Transport& transport);
 
 private:
     Vector<u8, 1024> m_data;
     Vector<NonnullRefPtr<AutoCloseFileDescriptor>, 1> m_fds;
+#ifdef AK_OS_WINDOWS
+    Vector<size_t> m_handle_offsets;
+#endif
 };
 
 enum class ErrorCode : u32 {

Libraries/LibIPC/MessageWindows.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
+ * Copyright (c) 2025, stasoid <stasoid@yahoo.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/ByteReader.h>
+#include <LibIPC/HandleType.h>
+#include <LibIPC/Message.h>
+
+#include <AK/Windows.h>
+
+namespace IPC {
+
+using MessageSizeType = u32;
+
+MessageBuffer::MessageBuffer()
+{
+    m_data.resize(sizeof(MessageSizeType));
+}
+
+ErrorOr<void> MessageBuffer::extend_data_capacity(size_t capacity)
+{
+    TRY(m_data.try_ensure_capacity(m_data.size() + capacity));
+    return {};
+}
+
+ErrorOr<void> MessageBuffer::append_data(u8 const* values, size_t count)
+{
+    TRY(m_data.try_append(values, count));
+    return {};
+}
+
+ErrorOr<void> MessageBuffer::append_file_descriptor(int handle)
+{
+    TRY(m_fds.try_append(adopt_ref(*new AutoCloseFileDescriptor(handle))));
+    TRY(m_handle_offsets.try_append(m_data.size()));
+
+    if (Core::System::is_socket(handle)) {
+        auto type = HandleType::Socket;
+        TRY(m_data.try_append(to_underlying(type)));
+
+        // The handle will be duplicated and WSAPROTOCOL_INFO will be filled later in TransportSocketWindows::transfer.
+        // It can't be duplicated here because it requires peer process pid, which only TransportSocketWindows knows about.
+        WSAPROTOCOL_INFO pi = {};
+        static_assert(sizeof(pi) >= sizeof(int));
+        ByteReader::store(reinterpret_cast<u8*>(&pi), handle);
+        TRY(m_data.try_append(reinterpret_cast<u8*>(&pi), sizeof(pi)));
+    } else {
+        auto type = HandleType::Generic;
+        TRY(m_data.try_append(to_underlying(type)));
+        // The handle will be overwritten by a duplicate handle later in TransportSocketWindows::transfer (for the same reason).
+        TRY(m_data.try_append(reinterpret_cast<u8*>(&handle), sizeof(handle)));
+    }
+    return {};
+}
+
+ErrorOr<void> MessageBuffer::transfer_message(Transport& transport)
+{
+    Checked<MessageSizeType> checked_message_size { m_data.size() };
+    checked_message_size -= sizeof(MessageSizeType);
+
+    if (checked_message_size.has_overflow())
+        return Error::from_string_literal("Message is too large for IPC encoding");
+
+    MessageSizeType const message_size = checked_message_size.value();
+    m_data.span().overwrite(0, reinterpret_cast<u8 const*>(&message_size), sizeof(message_size));
+
+    TRY(transport.transfer(m_data.span(), m_handle_offsets));
+    return {};
+}
+
+}

Libraries/LibIPC/Transport.h

@@ -10,6 +10,8 @@
 
 #if !defined(AK_OS_WINDOWS)
 #    include <LibIPC/TransportSocket.h>
+#else
+#    include <LibIPC/TransportSocketWindows.h>
 #endif
 
 namespace IPC {
@@ -18,7 +20,7 @@ namespace IPC {
 // Unix Domain Sockets
 using Transport = TransportSocket;
 #else
-#    error "LibIPC Transport has not been ported to this platform"
+using Transport = TransportSocketWindows;
 #endif
 
 }

Libraries/LibIPC/TransportSocketWindows.cpp

@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2024, Andrew Kaster <andrew@ladybird.org>
+ * Copyright (c) 2025, stasoid <stasoid@yahoo.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/ByteReader.h>
+#include <LibIPC/HandleType.h>
+#include <LibIPC/TransportSocketWindows.h>
+
+#include <AK/Windows.h>
+
+namespace IPC {
+
+TransportSocketWindows::TransportSocketWindows(NonnullOwnPtr<Core::LocalSocket> socket)
+    : m_socket(move(socket))
+{
+}
+
+void TransportSocketWindows::set_peer_pid(int pid)
+{
+    m_peer_pid = pid;
+}
+
+void TransportSocketWindows::set_up_read_hook(Function<void()> hook)
+{
+    VERIFY(m_socket->is_open());
+    m_socket->on_ready_to_read = move(hook);
+}
+
+bool TransportSocketWindows::is_open() const
+{
+    return m_socket->is_open();
+}
+
+void TransportSocketWindows::close()
+{
+    m_socket->close();
+}
+
+void TransportSocketWindows::wait_until_readable()
+{
+    auto readable = MUST(m_socket->can_read_without_blocking(-1));
+    VERIFY(readable);
+}
+
+ErrorOr<void> TransportSocketWindows::duplicate_handles(Bytes bytes, Vector<size_t> const& handle_offsets)
+{
+    if (handle_offsets.is_empty())
+        return {};
+
+    if (m_peer_pid == -1)
+        return Error::from_string_literal("Transport is not initialized");
+
+    HANDLE peer_process_handle = OpenProcess(PROCESS_DUP_HANDLE, FALSE, m_peer_pid);
+    if (!peer_process_handle)
+        return Error::from_windows_error();
+    ScopeGuard guard = [&] { CloseHandle(peer_process_handle); };
+
+    for (auto offset : handle_offsets) {
+
+        auto span = bytes.slice(offset);
+        if (span.size() < sizeof(HandleType))
+            return Error::from_string_literal("Not enough bytes");
+
+        UnderlyingType<HandleType> raw_type {};
+        ByteReader::load(span.data(), raw_type);
+        auto type = static_cast<HandleType>(raw_type);
+        if (type != HandleType::Generic && type != HandleType::Socket)
+            return Error::from_string_literal("Invalid handle type");
+        span = span.slice(sizeof(HandleType));
+
+        if (type == HandleType::Socket) {
+            if (span.size() < sizeof(WSAPROTOCOL_INFO))
+                return Error::from_string_literal("Not enough bytes for socket handle");
+
+            // We stashed the bytes of this process's version of the handle at the offset location
+            int handle = -1;
+            ByteReader::load(span.data(), handle);
+
+            auto* pi = reinterpret_cast<WSAPROTOCOL_INFO*>(span.data());
+            if (WSADuplicateSocket(handle, m_peer_pid, pi))
+                return Error::from_windows_error();
+        } else {
+            if (span.size() < sizeof(int))
+                return Error::from_string_literal("Not enough bytes for generic handle");
+
+            int handle = -1;
+            ByteReader::load(span.data(), handle);
+
+            HANDLE new_handle = INVALID_HANDLE_VALUE;
+            if (!DuplicateHandle(GetCurrentProcess(), to_handle(handle), peer_process_handle, &new_handle, 0, FALSE, DUPLICATE_SAME_ACCESS))
+                return Error::from_windows_error();
+
+            ByteReader::store(span.data(), to_fd(new_handle));
+        }
+    }
+
+    return {};
+}
+
+ErrorOr<void> TransportSocketWindows::transfer(Bytes bytes_to_write, Vector<size_t> const& handle_offsets)
+{
+    TRY(duplicate_handles(bytes_to_write, handle_offsets));
+
+    while (!bytes_to_write.is_empty()) {
+
+        ErrorOr<size_t> maybe_nwritten = m_socket->write_some(bytes_to_write);
+
+        if (maybe_nwritten.is_error()) {
+            auto error = maybe_nwritten.release_error();
+            if (error.code() != EWOULDBLOCK)
+                return error;
+
+            struct pollfd pollfd = {
+                .fd = static_cast<SOCKET>(m_socket->fd().value()),
+                .events = POLLOUT,
+                .revents = 0
+            };
+
+            auto result = WSAPoll(&pollfd, 1, -1);
+            if (result == 1)
+                continue;
+            if (result == SOCKET_ERROR)
+                return Error::from_windows_error();
+            VERIFY_NOT_REACHED();
+        }
+
+        bytes_to_write = bytes_to_write.slice(maybe_nwritten.value());
+    }
+    return {};
+}
+
+TransportSocketWindows::ReadResult TransportSocketWindows::read_as_much_as_possible_without_blocking(Function<void()> schedule_shutdown)
+{
+    ReadResult result;
+
+    while (is_open()) {
+
+        u8 buffer[4096];
+        auto maybe_bytes_read = m_socket->read_without_waiting({ buffer, sizeof(buffer) });
+
+        if (maybe_bytes_read.is_error()) {
+            auto error = maybe_bytes_read.release_error();
+            if (error.code() == EWOULDBLOCK)
+                break;
+            if (error.code() == ECONNRESET) {
+                schedule_shutdown();
+                break;
+            }
+            VERIFY_NOT_REACHED();
+        }
+
+        auto bytes_read = maybe_bytes_read.release_value();
+        if (bytes_read.is_empty()) {
+            schedule_shutdown();
+            break;
+        }
+
+        result.bytes.append(bytes_read.data(), bytes_read.size());
+    }
+
+    return result;
+}
+
+ErrorOr<int> TransportSocketWindows::release_underlying_transport_for_transfer()
+{
+    return m_socket->release_fd();
+}
+
+ErrorOr<IPC::File> TransportSocketWindows::clone_for_transfer()
+{
+    return IPC::File::clone_fd(m_socket->fd().value());
+}
+
+}

Libraries/LibIPC/TransportSocketWindows.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2024, Andrew Kaster <andrew@ladybird.org>
+ * Copyright (c) 2025, stasoid <stasoid@yahoo.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibCore/Socket.h>
+#include <LibIPC/File.h>
+
+namespace IPC {
+
+class TransportSocketWindows {
+    AK_MAKE_NONCOPYABLE(TransportSocketWindows);
+    AK_MAKE_DEFAULT_MOVABLE(TransportSocketWindows);
+
+public:
+    explicit TransportSocketWindows(NonnullOwnPtr<Core::LocalSocket> socket);
+
+    void set_peer_pid(int pid);
+    void set_up_read_hook(Function<void()>);
+    bool is_open() const;
+    void close();
+
+    void wait_until_readable();
+
+    ErrorOr<void> transfer(Bytes, Vector<size_t> const& handle_offsets);
+
+    struct [[nodiscard]] ReadResult {
+        Vector<u8> bytes;
+        Vector<int> fds; // always empty, present to avoid OS #ifdefs in Connection.cpp
+    };
+    ReadResult read_as_much_as_possible_without_blocking(Function<void()> schedule_shutdown);
+
+    // Obnoxious name to make it clear that this is a dangerous operation.
+    ErrorOr<int> release_underlying_transport_for_transfer();
+
+    ErrorOr<IPC::File> clone_for_transfer();
+
+private:
+    ErrorOr<void> duplicate_handles(Bytes, Vector<size_t> const& handle_offsets);
+
+private:
+    NonnullOwnPtr<Core::LocalSocket> m_socket;
+    int m_peer_pid = -1;
+};
+
+}