Fixes related to broken DH support in CMS

- DH support should work with both DH and DHX keys
- UKM parameter is optional so it can have length 0

Fixes #13810

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13869)

crypto/cms/cms_env.c

@@ -115,7 +115,7 @@ int cms_env_asn1_ctrl(CMS_RecipientInfo *ri, int cmd)
     } else
         return 0;
 
-    if (EVP_PKEY_is_a(pkey, "DHX"))
+    if (EVP_PKEY_is_a(pkey, "DHX") || EVP_PKEY_is_a(pkey, "DH"))
         return cms_dh_envelope(ri, cmd);
     else if (EVP_PKEY_is_a(pkey, "EC"))
         return cms_ecdh_envelope(ri, cmd);
@@ -1294,6 +1294,8 @@ int cms_pkey_get_ri_type(EVP_PKEY *pk)
     /* Check types that we know about */
     if (EVP_PKEY_is_a(pk, "DH"))
         return CMS_RECIPINFO_AGREE;
+    else if (EVP_PKEY_is_a(pk, "DHX"))
+        return CMS_RECIPINFO_AGREE;
     else if (EVP_PKEY_is_a(pk, "DSA"))
         return CMS_RECIPINFO_NONE;
     else if (EVP_PKEY_is_a(pk, "EC"))

crypto/evp/dh_ctrl.c

@@ -514,7 +514,7 @@ int EVP_PKEY_CTX_set0_dh_kdf_ukm(EVP_PKEY_CTX *ctx, unsigned char *ukm, int len)
     int ret;
     OSSL_PARAM params[2], *p = params;
 
-    if (len <= 0)
+    if (len < 0)
         return -1;
 
     ret = dh_param_derive_check(ctx);

crypto/evp/p_lib.c

@@ -972,6 +972,8 @@ int evp_pkey_name2type(const char *name)
         type = EVP_PKEY_DH;
     else if (strcasecmp(name, "X9.42 DH") == 0)
         type = EVP_PKEY_DHX;
+    else if (strcasecmp(name, "DHX") == 0)
+        type = EVP_PKEY_DHX;
     else if (strcasecmp(name, "DSA") == 0)
         type = EVP_PKEY_DSA;
 

test/recipes/80-test_cms.t

@@ -598,7 +598,7 @@ my @smime_cms_param_tests = (
         "-stream", "-out", "{output}.cms",
         "-recip", catfile($smdir, "smec1.pem"), "-aes-128-gcm", "-keyopt", "ecdh_kdf_md:sha256" ],
       [ "{cmd2}", "-decrypt", "-recip", catfile($smdir, "smec1.pem"),
-	      "-in", "{output}.cms", "-out", "{output}.txt" ],
+        "-in", "{output}.cms", "-out", "{output}.txt" ],
       \&final_compare
     ],
 
@@ -610,18 +610,16 @@ my @smime_cms_param_tests = (
       [ "{cmd2}", @prov, "-decrypt", "-recip", catfile($smdir, "smec2.pem"),
         "-in", "{output}.cms", "-out", "{output}.txt" ],
       \&final_compare
-    ]
+    ],
 
-    # TODO(3.0) Add this test back in when "dhpublicnumber" is supported
-    # in the keymanger.
-    #[ "enveloped content test streaming S/MIME format, X9.42 DH",
-    #  [ "{cmd1}", @prov, "-encrypt", "-in", $smcont,
-    #    "-stream", "-out", "{output}.cms",
-    #    "-recip", catfile($smdir, "smdh.pem"), "-aes128" ],
-    #  [ "{cmd2}", "-decrypt", "-recip", catfile($smdir, "smdh.pem"),
-    #    "-in", "{output}.cms", "-out", "{output}.txt" ],
-    #  \&final_compare
-    #]
+    [ "enveloped content test streaming S/MIME format, X9.42 DH",
+      [ "{cmd1}", @prov, "-encrypt", "-in", $smcont,
+        "-stream", "-out", "{output}.cms",
+        "-recip", catfile($smdir, "smdh.pem"), "-aes128" ],
+      [ "{cmd2}", "-decrypt", "-recip", catfile($smdir, "smdh.pem"),
+        "-in", "{output}.cms", "-out", "{output}.txt" ],
+      \&final_compare
+    ]
 );
 
 my @contenttype_cms_test = (