[crypto] Atomize all traces of MbedTLS, and require OpenSSL 3+ (#3606)

Closes #3137
Closes #3465

- Replace all mbedtls usage with OpenSSL
- require OpenSSL
- Up OpenSSL version to 3, cuz that's what we actually need...

CAVEATS:
- httplib also now required
- other ssl backends for svc are unused, maybe remove later
  * To be fair, our CI never used them anyways. And we never tested those

TESTERS PLEASE TEST:
- All games and applets boot
- Boot, load, exit, etc. times

Co-authored-by: crueter <crueter@eden-emu.dev>
Signed-off-by: lizzie <lizzie@eden-emu.dev>
Co-authored-by: crueter <crueter@eden-emu.dev>
Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/3606
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>
Reviewed-by: DraVee <dravee@eden-emu.dev>
Co-authored-by: lizzie <lizzie@eden-emu.dev>
Co-committed-by: lizzie <lizzie@eden-emu.dev>
This commit is contained in:
lizzie 2026-02-23 02:50:13 +01:00 committed by crueter
parent 80d6172084
commit 0a687b82d4
No known key found for this signature in database
GPG key ID: 425ACD2D4830EBC6
24 changed files with 372 additions and 393 deletions

View file

@ -1,13 +1,13 @@
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <array>
#include <cstring>
#include <mbedtls/cipher.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include "common/assert.h"
#include "common/logging/log.h"
#include "core/crypto/aes_util.h"
@ -28,83 +28,121 @@ NintendoTweak CalculateNintendoTweak(std::size_t sector_id) {
}
} // Anonymous namespace
static_assert(static_cast<std::size_t>(Mode::CTR) ==
static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_CTR),
"CTR has incorrect value.");
static_assert(static_cast<std::size_t>(Mode::ECB) ==
static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_ECB),
"ECB has incorrect value.");
static_assert(static_cast<std::size_t>(Mode::XTS) ==
static_cast<std::size_t>(MBEDTLS_CIPHER_AES_128_XTS),
"XTS has incorrect value.");
// Structure to hide mbedtls types from header file
// Structure to hide OpenSSL types from header file
struct CipherContext {
mbedtls_cipher_context_t encryption_context;
mbedtls_cipher_context_t decryption_context;
EVP_CIPHER_CTX* encryption_context = nullptr;
EVP_CIPHER_CTX* decryption_context = nullptr;
EVP_CIPHER* cipher = nullptr;
};
static inline const std::string GetCipherName(Mode mode, u32 key_size) {
std::string cipher;
std::size_t effective_bits = key_size * 8;
switch (mode) {
case Mode::CTR:
cipher = "CTR";
break;
case Mode::ECB:
cipher = "ECB";
break;
case Mode::XTS:
cipher = "XTS";
effective_bits /= 2;
break;
default:
UNREACHABLE();
}
return fmt::format("AES-{}-{}", effective_bits, cipher);
};
static EVP_CIPHER *GetCipher(Mode mode, u32 key_size) {
static auto fetch_cipher = [](Mode m, u32 k) {
return EVP_CIPHER_fetch(nullptr, GetCipherName(m, k).c_str(), nullptr);
};
static const struct {
EVP_CIPHER* ctr_16 = fetch_cipher(Mode::CTR, 16);
EVP_CIPHER* ecb_16 = fetch_cipher(Mode::ECB, 16);
EVP_CIPHER* xts_16 = fetch_cipher(Mode::XTS, 16);
EVP_CIPHER* ctr_32 = fetch_cipher(Mode::CTR, 32);
EVP_CIPHER* ecb_32 = fetch_cipher(Mode::ECB, 32);
EVP_CIPHER* xts_32 = fetch_cipher(Mode::XTS, 32);
} ciphers = {};
switch (mode) {
case Mode::CTR:
return key_size == 16 ? ciphers.ctr_16 : ciphers.ctr_32;
case Mode::ECB:
return key_size == 16 ? ciphers.ecb_16 : ciphers.ecb_32;
case Mode::XTS:
return key_size == 16 ? ciphers.xts_16 : ciphers.xts_32;
default:
UNIMPLEMENTED();
}
return nullptr;
}
// TODO: WHY TEMPLATE???????
template <typename Key, std::size_t KeySize>
Crypto::AESCipher<Key, KeySize>::AESCipher(Key key, Mode mode)
: ctx(std::make_unique<CipherContext>()) {
mbedtls_cipher_init(&ctx->encryption_context);
mbedtls_cipher_init(&ctx->decryption_context);
ASSERT_MSG((mbedtls_cipher_setup(
&ctx->encryption_context,
mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode))) ||
mbedtls_cipher_setup(
&ctx->decryption_context,
mbedtls_cipher_info_from_type(static_cast<mbedtls_cipher_type_t>(mode)))) == 0,
"Failed to initialize mbedtls ciphers.");
ctx->encryption_context = EVP_CIPHER_CTX_new();
ctx->decryption_context = EVP_CIPHER_CTX_new();
ctx->cipher = GetCipher(mode, KeySize);
if (ctx->cipher) {
EVP_CIPHER_up_ref(ctx->cipher);
} else {
UNIMPLEMENTED();
}
ASSERT(
!mbedtls_cipher_setkey(&ctx->encryption_context, key.data(), KeySize * 8, MBEDTLS_ENCRYPT));
ASSERT(
!mbedtls_cipher_setkey(&ctx->decryption_context, key.data(), KeySize * 8, MBEDTLS_DECRYPT));
//"Failed to set key on mbedtls ciphers.");
ASSERT_MSG(ctx->encryption_context && ctx->decryption_context && ctx->cipher,
"OpenSSL cipher context failed init!");
// now init ciphers
ASSERT(EVP_CipherInit_ex2(ctx->encryption_context, ctx->cipher, key.data(), NULL, 1, NULL));
ASSERT(EVP_CipherInit_ex2(ctx->decryption_context, ctx->cipher, key.data(), NULL, 0, NULL));
EVP_CIPHER_CTX_set_padding(ctx->encryption_context, 0);
EVP_CIPHER_CTX_set_padding(ctx->decryption_context, 0);
}
template <typename Key, std::size_t KeySize>
AESCipher<Key, KeySize>::~AESCipher() {
mbedtls_cipher_free(&ctx->encryption_context);
mbedtls_cipher_free(&ctx->decryption_context);
EVP_CIPHER_CTX_free(ctx->encryption_context);
EVP_CIPHER_CTX_free(ctx->decryption_context);
EVP_CIPHER_free(ctx->cipher);
}
template <typename Key, std::size_t KeySize>
void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* dest, Op op) const {
auto* const context = op == Op::Encrypt ? &ctx->encryption_context : &ctx->decryption_context;
mbedtls_cipher_reset(context);
auto* const context = op == Op::Encrypt ? ctx->encryption_context : ctx->decryption_context;
if (size == 0)
return;
const auto mode = mbedtls_cipher_get_cipher_mode(context);
std::size_t written = 0;
// reset
ASSERT(EVP_CipherInit_ex(context, nullptr, nullptr, nullptr, nullptr, -1));
if (mode != MBEDTLS_MODE_ECB) {
const int ret = mbedtls_cipher_update(context, src, size, dest, &written);
ASSERT(ret == 0);
if (written != size) {
LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", size, written);
}
return;
}
const auto block_size = mbedtls_cipher_get_block_size(context);
ASSERT(block_size <= AesBlockBytes);
const int block_size = EVP_CIPHER_CTX_get_block_size(context);
ASSERT(block_size > 0 && block_size <= int(AesBlockBytes));
const std::size_t whole_block_bytes = size - (size % block_size);
int written = 0;
if (whole_block_bytes != 0) {
const int ret = mbedtls_cipher_update(context, src, whole_block_bytes, dest, &written);
ASSERT(ret == 0);
if (written != whole_block_bytes) {
ASSERT(EVP_CipherUpdate(context, dest, &written, src, static_cast<int>(whole_block_bytes)));
if (std::size_t(written) != whole_block_bytes) {
LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.",
whole_block_bytes, written);
}
}
// tail
const std::size_t tail = size - whole_block_bytes;
if (tail == 0)
return;
@ -112,13 +150,13 @@ void AESCipher<Key, KeySize>::Transcode(const u8* src, std::size_t size, u8* des
std::array<u8, AesBlockBytes> tail_buffer{};
std::memcpy(tail_buffer.data(), src + whole_block_bytes, tail);
std::size_t tail_written = 0;
const int ret = mbedtls_cipher_update(context, tail_buffer.data(), block_size, tail_buffer.data(),
&tail_written);
ASSERT(ret == 0);
int tail_written = 0;
ASSERT(EVP_CipherUpdate(context, tail_buffer.data(), &tail_written, tail_buffer.data(), block_size));
if (tail_written != block_size) {
LOG_WARNING(Crypto, "Not all data was processed requested={:016X}, actual={:016X}.", block_size,
tail_written);
LOG_WARNING(Crypto, "Tail block not fully processed requested={:016X}, actual={:016X}.",
block_size, tail_written);
}
std::memcpy(dest + whole_block_bytes, tail_buffer.data(), tail);
@ -137,9 +175,10 @@ void AESCipher<Key, KeySize>::XTSTranscode(const u8* src, std::size_t size, u8*
template <typename Key, std::size_t KeySize>
void AESCipher<Key, KeySize>::SetIV(std::span<const u8> data) {
ASSERT_MSG((mbedtls_cipher_set_iv(&ctx->encryption_context, data.data(), data.size()) ||
mbedtls_cipher_set_iv(&ctx->decryption_context, data.data(), data.size())) == 0,
"Failed to set IV on mbedtls ciphers.");
const int ret_enc = EVP_CipherInit_ex(ctx->encryption_context, nullptr, nullptr, nullptr, data.data(), -1);
const int ret_dec = EVP_CipherInit_ex(ctx->decryption_context, nullptr, nullptr, nullptr, data.data(), -1);
ASSERT_MSG(ret_enc == 1 && ret_dec == 1, "Failed to set IV on OpenSSL contexts");
}
template class AESCipher<Key128>;