LibCrypto: Make PKSystem methods return a ByteBuffer directly

It used to be that the caller would supply a buffer to write the output
to. This created an anti-pattern in multiple places where the caller
would allocate a `ByteBuffer` and then use `.bytes()` to provide it to
the `PKSystem` method. Then the callee would resize the output buffer
and reassign it, but because the resize was on `Bytes` and not on
`ByteBuffer`, the caller using the latter would cause a bug.

Additionally, in pretty much all cases the buffer was pre-allocated
shortly before.
This commit is contained in:
devgianlu 2024-12-25 22:04:38 +01:00 committed by Ali Mohammad Pur
commit 0fc02d4d00
Notes: github-actions[bot] 2025-01-13 16:02:16 +00:00
9 changed files with 69 additions and 112 deletions

View file

@ -191,11 +191,7 @@ void TLSv12::build_rsa_pre_master_secret(PacketBuilder& builder)
}
Crypto::PK::RSA_PKCS1_EME rsa(certificate.public_key.rsa);
Vector<u8, 32> out;
out.resize(rsa.output_size());
auto outbuf = out.span();
MUST(rsa.encrypt(m_context.premaster_key, outbuf));
auto outbuf = MUST(rsa.encrypt(m_context.premaster_key));
if constexpr (TLS_DEBUG) {
dbgln("Encrypted: ");

View file

@ -381,14 +381,7 @@ ssize_t TLSv12::verify_rsa_server_key_exchange(ReadonlyBytes server_key_info_buf
Crypto::PK::RSAPrivateKey dummy_private_key;
auto rsa = Crypto::PK::RSA(certificate_public_key.rsa, dummy_private_key);
auto signature_verify_buffer_result = ByteBuffer::create_uninitialized(signature_length);
if (signature_verify_buffer_result.is_error()) {
dbgln("verify_rsa_server_key_exchange failed: Not enough memory");
return (i8)Error::OutOfMemory;
}
auto signature_verify_buffer = signature_verify_buffer_result.release_value();
auto signature_verify_bytes = signature_verify_buffer.bytes();
MUST(rsa.verify(signature, signature_verify_bytes));
auto signature_verify = MUST(rsa.verify(signature));
auto message_result = ByteBuffer::create_uninitialized(64 + server_key_info_buffer.size());
if (message_result.is_error()) {
@ -420,7 +413,7 @@ ssize_t TLSv12::verify_rsa_server_key_exchange(ReadonlyBytes server_key_info_buf
}
auto pkcs1 = Crypto::PK::EMSA_PKCS1_V1_5<Crypto::Hash::Manager>(hash_kind);
auto verification = pkcs1.verify(message, signature_verify_bytes, signature_length * 8);
auto verification = pkcs1.verify(message, signature_verify, signature_length * 8);
if (verification == Crypto::VerificationConsistency::Inconsistent) {
dbgln("verify_rsa_server_key_exchange failed: Verification of signature inconsistent");

View file

@ -345,18 +345,11 @@ bool Context::verify_certificate_pair(Certificate const& subject, Certificate co
Crypto::PK::RSAPrivateKey dummy_private_key;
Crypto::PK::RSAPublicKey public_key_copy { issuer.public_key.rsa };
auto rsa = Crypto::PK::RSA(public_key_copy, dummy_private_key);
auto verification_buffer_result = ByteBuffer::create_uninitialized(subject.signature_value.size());
if (verification_buffer_result.is_error()) {
dbgln("verify_certificate_pair: Unable to allocate buffer for verification");
return false;
}
auto verification_buffer = verification_buffer_result.release_value();
auto verification_buffer_bytes = verification_buffer.bytes();
MUST(rsa.verify(subject.signature_value, verification_buffer_bytes));
auto verification_bytes = MUST(rsa.verify(subject.signature_value));
ReadonlyBytes message = subject.tbs_asn1.bytes();
auto pkcs1 = Crypto::PK::EMSA_PKCS1_V1_5<Crypto::Hash::Manager>(kind);
auto verification = pkcs1.verify(message, verification_buffer_bytes, subject.signature_value.size() * 8);
auto verification = pkcs1.verify(message, verification_bytes, subject.signature_value.size() * 8);
return verification == Crypto::VerificationConsistency::Consistent;
}