hcrypto: Use builtin 1DES/RC2/RC4 (OpenSSL 3)

At some point before we make an 8.0 release we'll probably just remove
all the legacy, weak ciphers and hashes (except MD5, most likely).

To drop these we'll have to re-generate PKCS#12 test samples using
stronger PBEs, and possible add new PBE types.

lib/hcrypto/evp-openssl.c

@@ -80,6 +80,24 @@
 #define EVP_MD_CTX_free EVP_MD_CTX_destroy
 #endif
 
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+int _heim_openssl_fips_enabled(void);
+int
+_heim_openssl_fips_enabled(void)
+{
+    static int fips_enabled_res = -1;
+
+    if (fips_enabled_res != -1)
+        return fips_enabled_res;
+
+#ifdef HAVE_OPENSSL_30
+    return fips_enabled_res = !!EVP_default_properties_is_fips_enabled(NULL);
+#else
+    return fips_enabled_res = !!FIPS_mode();
+#endif
+}
+#endif
+
 /* A HEIM_BASE_ONCE argument struct for per-EVP one-time initialization */
 struct once_init_cipher_ctx {
     const hc_EVP_CIPHER **hc_memoizep;
@@ -438,7 +456,9 @@ OSSL_CIPHER_ALGORITHM(des_ede3_cbc, hc_EVP_CIPH_CBC_MODE)
  *
  * @ingroup hcrypto_evp
  */
+#ifndef HAVE_OPENSSL_30
 OSSL_CIPHER_ALGORITHM(des_cbc, hc_EVP_CIPH_CBC_MODE)
+#endif
 
 /**
  * The AES-128 cipher type (OpenSSL provider)
@@ -494,6 +514,7 @@ OSSL_CIPHER_ALGORITHM(aes_192_cfb8, hc_EVP_CIPH_CFB8_MODE)
  */
 OSSL_CIPHER_ALGORITHM(aes_256_cfb8, hc_EVP_CIPH_CFB8_MODE)
 
+#ifndef HAVE_OPENSSL_30
 /*
  * RC2 is only needed for tests of PKCS#12 support, which currently uses
  * the RC2 PBE.  So no RC2 -> tests fail.
@@ -530,6 +551,7 @@ OSSL_CIPHER_ALGORITHM(rc2_40_cbc,
 OSSL_CIPHER_ALGORITHM(rc2_64_cbc,
                       hc_EVP_CIPH_CBC_MODE |
                       hc_EVP_CIPH_VARIABLE_LENGTH)
+#endif
 
 /**
  * The Camellia-128 cipher type - OpenSSL
@@ -558,6 +580,7 @@ OSSL_CIPHER_ALGORITHM(camellia_192_cbc, hc_EVP_CIPH_CBC_MODE)
  */
 OSSL_CIPHER_ALGORITHM(camellia_256_cbc, hc_EVP_CIPH_CBC_MODE)
 
+#ifndef HAVE_OPENSSL_30
 /**
  * The RC4 cipher type (OpenSSL provider)
  *
@@ -588,6 +611,7 @@ OSSL_CIPHER_ALGORITHM(rc4_40,
  * @ingroup hcrypto_evp
  */
 OSSL_MD_ALGORITHM(md4)
+#endif
 
 /**
  * The MD5 hash algorithm (OpenSSL provider)

lib/hcrypto/evp.c

@@ -59,6 +59,7 @@
 #  define HCRYPTO_DEF_PROVIDER pkcs11_hcrypto
 # elif HAVE_HCRYPTO_W_OPENSSL
 #  define HCRYPTO_DEF_PROVIDER ossl
+#  define HCRYPTO_DEF_PROVIDER_IS_OPENSSL
 # else
 #  define HCRYPTO_DEF_PROVIDER hcrypto
 # endif
@@ -69,6 +70,11 @@
 
 #define EVP_DEF_OP(_prov,_op) HC_CONCAT4(EVP_,_prov,_,_op)()
 
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+extern int _heim_openssl_fips_enabled(void);
+#endif
+
+
 /**
  * @page page_evp EVP - generic crypto interface
  *
@@ -463,6 +469,13 @@ const EVP_MD *
 EVP_md4(void) HC_DEPRECATED_CRYPTO
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, md4);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, md4);
 }
 
@@ -1049,6 +1062,13 @@ const EVP_CIPHER *
 EVP_rc2_cbc(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, rc2_cbc);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, rc2_cbc);
 }
 
@@ -1064,6 +1084,13 @@ const EVP_CIPHER *
 EVP_rc2_40_cbc(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, rc2_40_cbc);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, rc2_40_cbc);
 }
 
@@ -1079,6 +1106,13 @@ const EVP_CIPHER *
 EVP_rc2_64_cbc(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, rc2_64_cbc);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, rc2_64_cbc);
 }
 
@@ -1094,6 +1128,13 @@ const EVP_CIPHER *
 EVP_rc4(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, rc4);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, rc4);
 }
 
@@ -1109,6 +1150,13 @@ const EVP_CIPHER *
 EVP_rc4_40(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, rc4_40);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, rc4_40);
 }
 
@@ -1124,6 +1172,13 @@ const EVP_CIPHER *
 EVP_des_cbc(void)
 {
     hcrypto_validate();
+#if defined(HCRYPTO_DEF_PROVIDER_IS_OPENSSL) && defined(HAVE_OPENSSL_30)
+#if defined(HAVE_OPENSSL_FIPS_H) || defined(HAVE_OPENSSL_FIPS_MODE_SET_API)
+    if (_heim_openssl_fips_enabled())
+        return NULL;
+#endif
+    return EVP_DEF_OP(hcrypto, des_cbc);
+#endif
     return EVP_DEF_OP(HCRYPTO_DEF_PROVIDER, des_cbc);
 }
 

lib/hcrypto/evp.h

@@ -193,7 +193,7 @@ struct hc_CIPHER_CTX {
     unsigned long flags;
     void *cipher_data;
     int final_used;
-    int block_mask;
+    unsigned block_mask;
     unsigned char final[EVP_MAX_BLOCK_LENGTH];
 };
 

lib/hcrypto/test_cipher.c

@@ -422,14 +422,8 @@ main(int argc, char **argv)
 	ret += test_cipher(i, EVP_ossl_aes_256_cbc(), &aes_tests[i]);
     for (i = 0; i < sizeof(aes_cfb_tests)/sizeof(aes_cfb_tests[0]); i++)
 	ret += test_cipher(i, EVP_ossl_aes_128_cfb8(), &aes_cfb_tests[i]);
-    for (i = 0; i < sizeof(rc2_tests)/sizeof(rc2_tests[0]); i++)
-	ret += test_cipher(i, EVP_ossl_rc2_cbc(), &rc2_tests[i]);
-    for (i = 0; i < sizeof(rc2_40_tests)/sizeof(rc2_40_tests[0]); i++)
-	ret += test_cipher(i, EVP_ossl_rc2_40_cbc(), &rc2_40_tests[i]);
     for (i = 0; i < sizeof(des_ede3_tests)/sizeof(des_ede3_tests[0]); i++)
 	ret += test_cipher(i, EVP_ossl_des_ede3_cbc(), &des_ede3_tests[i]);
-    for (i = 0; i < sizeof(rc4_tests)/sizeof(rc4_tests[0]); i++)
-	ret += test_cipher(i, EVP_ossl_rc4(), &rc4_tests[i]);
 #endif /* PKCS11_MODULE_PATH */
 
     return ret;

lib/hcrypto/test_crypto.in

@@ -82,11 +82,11 @@ for a in unix fortuna egd w32crypto ;do
 	    { echo "rand output same!" ; exit 1; }
 done
 
-./example_evp_cipher 1 ${srcdir}/test_crypto.in test-out-1 || \
-    { echo "1 failed" ; exit 1; }
-
-for a in 7 15 16 17 31 32 33 ; do
-    ./example_evp_cipher $a ${srcdir}/test_crypto.in test-out-$a
+for a in 1 7 15 16 17 31 32 33 ; do
+    ./example_evp_cipher $a ${srcdir}/test_crypto.in test-out-$a ||
+        { echo "$s failed" ; exit 1; }
+done
+for a in   7 15 16 17 31 32 33 ; do
     cmp test-out-1 test-out-$a || { echo "cmp $a failed" ; exit 1; }
 done