Merge pull request #9592 from julek-wolfssl/fix/coverity-tls-frag

Fix Coverity (D)TLS fragmentation size checks

Author: dgarske

src/dtls13.c

@@ -978,7 +978,8 @@ static int Dtls13SendOneFragmentRtx(WOLFSSL* ssl,
 static int Dtls13SendFragmentedInternal(WOLFSSL* ssl)
 {
     int fragLength, rlHeaderLength;
-    int remainingSize, maxFragment;
+    word32 remainingSize;
+    int maxFragment;
     int recordLength, outputSz;
     byte isEncrypted;
     byte* output;
@@ -988,16 +989,19 @@ static int Dtls13SendFragmentedInternal(WOLFSSL* ssl)
         (enum HandShakeType)ssl->dtls13FragHandshakeType);
     rlHeaderLength = Dtls13GetRlHeaderLength(ssl, isEncrypted);
     maxFragment = wolfssl_local_GetMaxPlaintextSize(ssl);
-
+    if (maxFragment <= DTLS_HANDSHAKE_HEADER_SZ ||
+            maxFragment > MAX_RECORD_SIZE ||
+            ssl->dtls13FragOffset > ssl->dtls13MessageLength) {
+        Dtls13FreeFragmentsBuffer(ssl);
+        return BUFFER_E;
+    }
     remainingSize = ssl->dtls13MessageLength - ssl->dtls13FragOffset;
 
     while (remainingSize > 0) {
 
         fragLength = maxFragment - DTLS_HANDSHAKE_HEADER_SZ;
-
-        if (fragLength > remainingSize) {
-            fragLength = remainingSize;
-        }
+        if (fragLength > (int)remainingSize)
+            fragLength = (int)remainingSize;
 
         recordLength = fragLength + rlHeaderLength + DTLS_HANDSHAKE_HEADER_SZ;
         outputSz = wolfssl_local_GetRecordSize(ssl,
@@ -1041,7 +1045,7 @@ static int Dtls13SendFragmentedInternal(WOLFSSL* ssl)
         }
 
         ssl->dtls13FragOffset += fragLength;
-        remainingSize -= fragLength;
+        remainingSize -= (word32)fragLength;
     }
 
     /* we sent all fragments */

src/internal.c

@@ -10766,6 +10766,8 @@ static int SendHandshakeMsg(WOLFSSL* ssl, byte* input, word32 inputSz,
         maxFrag -= DTLS_HANDSHAKE_HEADER_SZ;
     }
 #endif
+    if (maxFrag <= 0 || maxFrag > MAX_RECORD_SIZE)
+        return BUFFER_E;
 
     /* Make sure input is not the ssl output buffer as this
      * function doesn't handle that */
@@ -10801,6 +10803,8 @@ static int SendHandshakeMsg(WOLFSSL* ssl, byte* input, word32 inputSz,
             fragSz = inputSz - ssl->fragOffset;
 
         /* check for available size */
+        if (fragSz > (word32)MAX_RECORD_SIZE)
+            return BUFFER_E;
         outputSz = headerSz + (int)fragSz;
         if (IsEncryptionOn(ssl, 1))
             outputSz += cipherExtraData(ssl);
@@ -10816,6 +10820,8 @@ static int SendHandshakeMsg(WOLFSSL* ssl, byte* input, word32 inputSz,
             int dataSz = (int)fragSz;
 #ifdef WOLFSSL_DTLS
             if (ssl->options.dtls) {
+                if (fragSz + DTLS_HANDSHAKE_HEADER_SZ > (word32)MAX_RECORD_SIZE)
+                    return BUFFER_E;
                 data   -= DTLS_HANDSHAKE_HEADER_SZ;
                 dataSz += DTLS_HANDSHAKE_HEADER_SZ;
                 AddHandShakeHeader(data, inputSz, ssl->fragOffset, fragSz,