Allow small RSA exponents in the default provider

Fixes #16255

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16285)

crypto/rsa/rsa_sp800_56b_check.c

@@ -218,30 +218,21 @@ int ossl_rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)
     return ret;
 }
 
-#ifndef FIPS_MODULE
-static int bn_is_three(const BIGNUM *bn)
-{
-    BIGNUM *num = BN_dup(bn);
-    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
-
-    BN_free(num);
-    return ret;
-}
-#endif /* FIPS_MODULE */
-
-/* Check exponent is odd, and has a bitlen ranging from [17..256] */
+/*
+ * Check exponent is odd.
+ * For FIPS also check the bit length is in the range [17..256]
+ */
 int ossl_rsa_check_public_exponent(const BIGNUM *e)
 {
+#ifdef FIPS_MODULE
     int bitlen;
 
-    /* For legacy purposes RSA_3 is allowed in non fips mode */
-#ifndef FIPS_MODULE
-    if (bn_is_three(e))
-        return 1;
-#endif /* FIPS_MODULE */
-
     bitlen = BN_num_bits(e);
     return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
+#else
+    /* Allow small exponents larger than 1 for legacy purposes */
+    return BN_is_odd(e) && BN_cmp(e, BN_value_one()) > 0;
+#endif /* FIPS_MODULE */
 }
 
 /*

test/rsa_sp800_56b_test.c

@@ -104,26 +104,29 @@ static BIGNUM *bn_load_new(const unsigned char *data, int sz)
     return ret;
 }
 
+/* Check that small rsa exponents are allowed in non FIPS mode */
 static int test_check_public_exponent(void)
 {
     int ret = 0;
     BIGNUM *e = NULL;
 
     ret = TEST_ptr(e = BN_new())
-          /* e is too small */
-          && TEST_true(BN_set_word(e, 65535))
+          /* e is too small will fail */
+          && TEST_true(BN_set_word(e, 1))
           && TEST_false(ossl_rsa_check_public_exponent(e))
           /* e is even will fail */
           && TEST_true(BN_set_word(e, 65536))
           && TEST_false(ossl_rsa_check_public_exponent(e))
           /* e is ok */
+          && TEST_true(BN_set_word(e, 3))
+          && TEST_true(ossl_rsa_check_public_exponent(e))
+          && TEST_true(BN_set_word(e, 17))
+          && TEST_true(ossl_rsa_check_public_exponent(e))
           && TEST_true(BN_set_word(e, 65537))
           && TEST_true(ossl_rsa_check_public_exponent(e))
-          /* e = 2^256 is too big */
+          /* e = 2^256 + 1 is ok */
           && TEST_true(BN_lshift(e, BN_value_one(), 256))
-          && TEST_false(ossl_rsa_check_public_exponent(e))
-          /* e = 2^256-1 is odd and in range */
-          && TEST_true(BN_sub(e, e, BN_value_one()))
+          && TEST_true(BN_add(e, e, BN_value_one()))
           && TEST_true(ossl_rsa_check_public_exponent(e));
     BN_free(e);
     return ret;