Avoid using functions deprecated in OpenSSL 3.0

Limelight/Crypto/CryptoManager.m

@@ -9,7 +9,6 @@
 #import "CryptoManager.h"
 #import "mkcert.h"
 
-#include <openssl/aes.h>
 #include <openssl/sha.h>
 #include <openssl/x509.h>
 #include <openssl/pem.h>
@@ -45,46 +44,47 @@ static NSData* p12 = nil;
 }
 
 - (NSData*) aesEncrypt:(NSData*)data withKey:(NSData*)key {
-    AES_KEY aesKey;
-    AES_set_encrypt_key([key bytes], 128, &aesKey);
-    int size = [self getEncryptSize:data];
-    unsigned char* buffer = malloc(size);
-    unsigned char* blockRoundedBuffer = calloc(1, size);
-    memcpy(blockRoundedBuffer, [data bytes], [data length]);
+    EVP_CIPHER_CTX* cipher;
+    int ciphertextLen;
+
+    cipher = EVP_CIPHER_CTX_new();
+
+    EVP_EncryptInit(cipher, EVP_aes_128_ecb(), [key bytes], NULL);
+    EVP_CIPHER_CTX_set_padding(cipher, 0);
+
+    NSMutableData* ciphertext = [NSMutableData dataWithLength:[data length]];
+    EVP_EncryptUpdate(cipher,
+                      [ciphertext mutableBytes],
+                      &ciphertextLen,
+                      [data bytes],
+                      (int)[data length]);
+    assert(ciphertextLen == [ciphertext length]);
+
+    EVP_CIPHER_CTX_free(cipher);
     
-    // AES_encrypt only encrypts the first 16 bytes so iterate the entire buffer
-    int blockOffset = 0;
-    while (blockOffset < size) {
-        AES_encrypt(blockRoundedBuffer + blockOffset, buffer + blockOffset, &aesKey);
-        blockOffset += 16;
-    }
-    
-    NSData* encryptedData = [NSData dataWithBytes:buffer length:size];
-    free(buffer);
-    free(blockRoundedBuffer);
-    return encryptedData;
+    return ciphertext;
 }
 
 - (NSData*) aesDecrypt:(NSData*)data withKey:(NSData*)key {
-    AES_KEY aesKey;
-    AES_set_decrypt_key([key bytes], 128, &aesKey);
-    unsigned char* buffer = malloc([data length]);
-    
-    // AES_decrypt only decrypts the first 16 bytes so iterate the entire buffer
-    int blockOffset = 0;
-    while (blockOffset < [data length]) {
-        AES_decrypt([data bytes] + blockOffset, buffer + blockOffset, &aesKey);
-        blockOffset += 16;
-    }
-    
-    NSData* decryptedData = [NSData dataWithBytes:buffer length:[data length]];
-    free(buffer);
-    return decryptedData;
-}
+    EVP_CIPHER_CTX* cipher;
+    int plaintextLen;
 
-- (int) getEncryptSize:(NSData*)data {
-    // the size is the length of the data ceiling to the nearest 16 bytes
-    return (((int)[data length] + 15) / 16) * 16;
+    cipher = EVP_CIPHER_CTX_new();
+
+    EVP_DecryptInit(cipher, EVP_aes_128_ecb(), [key bytes], NULL);
+    EVP_CIPHER_CTX_set_padding(cipher, 0);
+
+    NSMutableData* plaintext = [NSMutableData dataWithLength:[data length]];
+    EVP_DecryptUpdate(cipher,
+                      [plaintext mutableBytes],
+                      &plaintextLen,
+                      [data bytes],
+                      (int)[data length]);
+    assert(plaintextLen == [plaintext length]);
+
+    EVP_CIPHER_CTX_free(cipher);
+    
+    return plaintext;
 }
 
 + (NSData*) pemToDer:(NSData*)pemCertBytes {

Limelight/Crypto/mkcert.c

@@ -6,7 +6,6 @@
 
 #include <openssl/pem.h>
 #include <openssl/rsa.h>
-#include <openssl/bn.h>
 #include <openssl/x509.h>
 #include <openssl/rand.h>
 
@@ -16,15 +15,18 @@ static const int NUM_YEARS = 20;
 
 void mkcert(X509 **x509p, EVP_PKEY **pkeyp, int bits, int serial, int years) {
     X509* cert = X509_new();
-    EVP_PKEY* pk = EVP_PKEY_new();
-    BIGNUM* bne = BN_new();
-    RSA* rsa = RSA_new();
+    
+    EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
 
-    BN_set_word(bne, RSA_F4);
-    RSA_generate_key_ex(rsa, bits, bne, NULL);
+    EVP_PKEY_keygen_init(ctx);
+    EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits);
 
-    EVP_PKEY_assign_RSA(pk, rsa);
+    // pk must be initialized on input
+    EVP_PKEY* pk = NULL;
+    EVP_PKEY_keygen(ctx, &pk);
 
+    EVP_PKEY_CTX_free(ctx);
+    
     X509_set_version(cert, 2);
     ASN1_INTEGER_set(X509_get_serialNumber(cert), serial);
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
@@ -53,8 +55,6 @@ void mkcert(X509 **x509p, EVP_PKEY **pkeyp, int bits, int serial, int years) {
     X509_set_issuer_name(cert, name);
 
     X509_sign(cert, pk, EVP_sha256());
-
-    BN_free(bne);
     
     *x509p = cert;
     *pkeyp = pk;
@@ -66,7 +66,6 @@ struct CertKeyPair generateCertKeyPair(void) {
     EVP_PKEY *pkey = NULL;
     PKCS12 *p12 = NULL;
    
-    CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
     bio_err = BIO_new_fp(stderr, BIO_NOCLOSE);
     
     mkcert(&x509, &pkey, NUM_BITS, SERIAL, NUM_YEARS);