From 6e721110f931a7c1a136e5e7b08ec55854f6ac18 Mon Sep 17 00:00:00 2001
From: devgianlu <altomanigianluca@gmail.com>
Date: Wed, 25 Dec 2024 22:24:14 +0100
Subject: [PATCH] LibCrypto: Make RSA class easily configurable

This is a small change to allow subclasses of `RSA` to configure the
`EVP_PKEY_CTX` without rewriting everything.
---
 Libraries/LibCrypto/PK/RSA.cpp | 10 ++++++++--
 Libraries/LibCrypto/PK/RSA.h   |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Libraries/LibCrypto/PK/RSA.cpp b/Libraries/LibCrypto/PK/RSA.cpp
index 98d31c24be8..528f6f52f89 100644
--- a/Libraries/LibCrypto/PK/RSA.cpp
+++ b/Libraries/LibCrypto/PK/RSA.cpp
@@ -223,6 +223,12 @@ ErrorOr<OpenSSL_PKEY> RSA::private_key_to_openssl_pkey(PrivateKeyType const& pri
 
 #undef OPENSSL_SET_KEY_PARAM_NOT_ZERO
 
+ErrorOr<void> RSA::configure(OpenSSL_PKEY_CTX& ctx)
+{
+    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_NO_PADDING));
+    return {};
+}
+
 ErrorOr<ByteBuffer> RSA::encrypt(ReadonlyBytes in)
 {
     auto key = TRY(public_key_to_openssl_pkey(m_public_key));
@@ -230,7 +236,7 @@ ErrorOr<ByteBuffer> RSA::encrypt(ReadonlyBytes in)
     auto ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new_from_pkey(nullptr, key.ptr(), nullptr)));
 
     OPENSSL_TRY(EVP_PKEY_encrypt_init(ctx.ptr()));
-    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_NO_PADDING));
+    TRY(configure(ctx));
 
     size_t out_size = 0;
     OPENSSL_TRY(EVP_PKEY_encrypt(ctx.ptr(), nullptr, &out_size, in.data(), in.size()));
@@ -247,7 +253,7 @@ ErrorOr<ByteBuffer> RSA::decrypt(ReadonlyBytes in)
     auto ctx = TRY(OpenSSL_PKEY_CTX::wrap(EVP_PKEY_CTX_new_from_pkey(nullptr, key.ptr(), nullptr)));
 
     OPENSSL_TRY(EVP_PKEY_decrypt_init(ctx.ptr()));
-    OPENSSL_TRY(EVP_PKEY_CTX_set_rsa_padding(ctx.ptr(), RSA_NO_PADDING));
+    TRY(configure(ctx));
 
     size_t out_size = 0;
     OPENSSL_TRY(EVP_PKEY_decrypt(ctx.ptr(), nullptr, &out_size, in.data(), in.size()));
diff --git a/Libraries/LibCrypto/PK/RSA.h b/Libraries/LibCrypto/PK/RSA.h
index d5496524630..1500879b65d 100644
--- a/Libraries/LibCrypto/PK/RSA.h
+++ b/Libraries/LibCrypto/PK/RSA.h
@@ -222,6 +222,8 @@ public:
     void set_private_key(PrivateKeyType const& key) { m_private_key = key; }
 
 protected:
+    virtual ErrorOr<void> configure(OpenSSL_PKEY_CTX& ctx);
+
     static ErrorOr<OpenSSL_PKEY> public_key_to_openssl_pkey(PublicKeyType const& public_key);
     static ErrorOr<OpenSSL_PKEY> private_key_to_openssl_pkey(PrivateKeyType const& private_key);
 };