LibCrypto: Implement RSA_PSS_EMSA with OpenSSL

Author: https://github.com/devgianlu
Commit: 3eeb35e787
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/3252
Reviewed-by: https://github.com/gmta ✅

Libraries/LibCrypto/PK/RSA.cpp

@@ -454,4 +454,12 @@ ErrorOr<void> RSA_OAEP_EME::configure(OpenSSL_PKEY_CTX& ctx)
     return {};
 }
 
+ErrorOr<void> RSA_PSS_EMSA::configure(OpenSSL_PKEY_CTX& ctx)
+{
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_PKCS1_PSS_PADDING));
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_mgf1_md(ctx.ptr(), TRY(hash_kind_to_hash_type(m_hash_kind))));
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx.ptr(), m_salt_length.value_or(RSA_PSS_SALTLEN_MAX)));
+    return {};
+}
+
 }

Libraries/LibCrypto/PK/RSA.h

@@ -344,4 +344,28 @@ private:
     Optional<ReadonlyBytes> m_label {};
 };
 
+class RSA_PSS_EMSA : public RSA_EMSA {
+public:
+    template<typename... Args>
+    RSA_PSS_EMSA(Hash::HashKind hash_kind, Args... args)
+        : RSA_EMSA(hash_kind, args...)
+    {
+    }
+
+    ~RSA_PSS_EMSA() = default;
+
+    virtual ByteString class_name() const override
+    {
+        return "RSA_PSS-EMSA";
+    }
+
+    void set_salt_length(int value) { m_salt_length = value; }
+
+protected:
+    ErrorOr<void> configure(OpenSSL_PKEY_CTX& ctx) override;
+
+private:
+    Optional<int> m_salt_length;
+};
+
 }

Tests/LibCrypto/CMakeLists.txt

@@ -15,6 +15,7 @@ set(TEST_SOURCES
     TestOAEP.cpp
     TestPBKDF2.cpp
     TestPoly1305.cpp
+    TestPSS.cpp
     TestRSA.cpp
 )
 

Tests/LibCrypto/TestPSS.cpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2025, Altomani Gianluca <altomanigianluca@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/ByteBuffer.h>
+#include <LibCrypto/PK/RSA.h>
+#include <LibTest/TestCase.h>
+
+TEST_CASE(test_pss)
+{
+    auto msg = "WellHelloFriendsWellHelloFriendsWellHelloFriendsWellHelloFriends"sv.bytes();
+
+    auto keypair = TRY_OR_FAIL(Crypto::PK::RSA::generate_key_pair(1024));
+    auto rsa = Crypto::PK::RSA_PSS_EMSA(Crypto::Hash::HashKind::SHA1, keypair);
+    rsa.set_salt_length(48);
+
+    auto sig = TRY_OR_FAIL(rsa.sign(msg));
+    auto ok = TRY_OR_FAIL(rsa.verify(msg, sig));
+    EXPECT(ok);
+}
+
+TEST_CASE(test_pss_tampered_message)
+{
+    auto msg = "WellHelloFriendsWellHelloFriendsWellHelloFriendsWellHelloFriends"sv.bytes();
+    auto msg_tampered = "WellHell0FriendsWellHelloFriendsWellHelloFriendsWellHelloFriends"sv.bytes();
+
+    auto keypair = TRY_OR_FAIL(Crypto::PK::RSA::generate_key_pair(1024));
+    auto rsa = Crypto::PK::RSA_PSS_EMSA(Crypto::Hash::HashKind::SHA1, keypair);
+    rsa.set_salt_length(48);
+
+    auto sig = TRY_OR_FAIL(rsa.sign(msg));
+    auto ok = TRY_OR_FAIL(rsa.verify(msg_tampered, sig));
+    EXPECT(!ok);
+}
+
+TEST_CASE(test_pss_tampered_signature)
+{
+    auto msg = "WellHelloFriendsWellHelloFriendsWellHelloFriendsWellHelloFriends"sv.bytes();
+
+    auto keypair = TRY_OR_FAIL(Crypto::PK::RSA::generate_key_pair(1024));
+    auto rsa = Crypto::PK::RSA_PSS_EMSA(Crypto::Hash::HashKind::SHA1, keypair);
+    rsa.set_salt_length(48);
+
+    auto sig = TRY_OR_FAIL(rsa.sign(msg));
+
+    // Tamper with the signature
+    sig[8]++;
+
+    auto ok = TRY_OR_FAIL(rsa.verify(msg, sig));
+    EXPECT(!ok);
+}