Don't overflow the output length in EVP_CipherUpdate calls

CVE-2021-23840

Reviewed-by: Paul Dale <pauli@openssl.org>

crypto/err/openssl.txt

@@ -708,6 +708,7 @@ EVP_R_ONLY_ONESHOT_SUPPORTED:177:only oneshot supported
 EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE:150:\
 	operation not supported for this keytype
 EVP_R_OPERATON_NOT_INITIALIZED:151:operation not initialized
+EVP_R_OUTPUT_WOULD_OVERFLOW:202:output would overflow
 EVP_R_PARAMETER_TOO_LARGE:187:parameter too large
 EVP_R_PARTIALLY_OVERLAPPING:162:partially overlapping buffers
 EVP_R_PBKDF2_ERROR:181:pbkdf2 error

crypto/evp/evp_enc.c

@@ -11,6 +11,7 @@
 #define OPENSSL_SUPPRESS_DEPRECATED
 
 #include <stdio.h>
+#include <limits.h>
 #include <assert.h>
 #include "internal/cryptlib.h"
 #include <openssl/evp.h>
@@ -511,6 +512,18 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx,
             return 1;
         } else {
             j = bl - i;
+
+            /*
+             * Once we've processed the first j bytes from in, the amount of
+             * data left that is a multiple of the block length is:
+             * (inl - j) & ~(bl - 1)
+             * We must ensure that this amount of data, plus the one block that
+             * we process from ctx->buf does not exceed INT_MAX
+             */
+            if (((inl - j) & ~(bl - 1)) > INT_MAX - bl) {
+                ERR_raise(ERR_LIB_EVP, EVP_R_OUTPUT_WOULD_OVERFLOW);
+                return 0;
+            }
             memcpy(&(ctx->buf[i]), in, j);
             inl -= j;
             in += j;
@@ -771,6 +784,19 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
             ERR_raise(ERR_LIB_EVP, EVP_R_PARTIALLY_OVERLAPPING);
             return 0;
         }
+        /*
+         * final_used is only ever set if buf_len is 0. Therefore the maximum
+         * length output we will ever see from evp_EncryptDecryptUpdate is
+         * the maximum multiple of the block length that is <= inl, or just:
+         * inl & ~(b - 1)
+         * Since final_used has been set then the final output length is:
+         * (inl & ~(b - 1)) + b
+         * This must never exceed INT_MAX
+         */
+        if ((inl & ~(b - 1)) > INT_MAX - b) {
+            ERR_raise(ERR_LIB_EVP, EVP_R_OUTPUT_WOULD_OVERFLOW);
+            return 0;
+        }
         memcpy(out, ctx->final, b);
         out += b;
         fix_len = 1;

crypto/evp/evp_err.c

@@ -137,6 +137,8 @@ static const ERR_STRING_DATA EVP_str_reasons[] = {
     "operation not supported for this keytype"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_OPERATON_NOT_INITIALIZED),
     "operation not initialized"},
+    {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_OUTPUT_WOULD_OVERFLOW),
+    "output would overflow"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_PARAMETER_TOO_LARGE),
     "parameter too large"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_PARTIALLY_OVERLAPPING),
@@ -153,7 +155,7 @@ static const ERR_STRING_DATA EVP_str_reasons[] = {
     "set default property failure"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_TOO_MANY_RECORDS), "too many records"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNABLE_TO_ENABLE_LOCKING),
-    "unable to enable parent locking"},
+    "unable to enable locking"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNABLE_TO_GET_MAXIMUM_REQUEST_SIZE),
     "unable to get maximum request size"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_UNABLE_TO_GET_RANDOM_STRENGTH),

include/crypto/evperr.h

@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 2020-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy

include/openssl/evperr.h

@@ -97,6 +97,7 @@
 # define EVP_R_ONLY_ONESHOT_SUPPORTED                     177
 # define EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE   150
 # define EVP_R_OPERATON_NOT_INITIALIZED                   151
+# define EVP_R_OUTPUT_WOULD_OVERFLOW                      202
 # define EVP_R_PARAMETER_TOO_LARGE                        187
 # define EVP_R_PARTIALLY_OVERLAPPING                      162
 # define EVP_R_PBKDF2_ERROR                               181