allow config for max body size by content-type

Author: robotdan

src/main/java/io/fusionauth/http/server/Configurable.java

@@ -17,6 +17,7 @@
 
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.Map;
 
 import io.fusionauth.http.io.MultipartConfiguration;
 import io.fusionauth.http.log.LoggerFactory;
@@ -168,20 +169,6 @@ default T withLoggerFactory(LoggerFactory loggerFactory) {
     return (T) this;
   }
 
-  /**
-   * The maximum size of the HTTP request body in bytes when the Content-Type is application/x-www-form-urlencoded. Defaults to 10
-   * Megabytes.
-   * <p>
-   * Set this to -1 to disable this limitation.
-   *
-   * @param maxFormDataSize the maximum size of the HTTP request body in bytes.
-   * @return This.
-   */
-  default T withMaxFormDataSize(int maxFormDataSize) {
-    configuration().withMaxFormDataSize(maxFormDataSize);
-    return (T) this;
-  }
-
   /**
    * Sets the maximum number of pending socket connections per HTTP listener.
    * <p>
@@ -199,14 +186,15 @@ default T withMaxPendingSocketConnections(int maxPendingSocketConnections) {
   }
 
   /**
-   * Sets the maximum size of the HTTP request body. If this limit is exceeded, the connection will be closed. Defaults to 128 Megabytes.
+   * Sets the maximum size of the HTTP request body by optionally per Content-Type. If this limit is exceeded, the connection will be closed.
+   * Defaults to 128 Megabytes.
    * <p>
    * Set this to -1 to disable this limitation.
    *
-   * @param maxRequestBodySize the maximum size in bytes for the HTTP request body
+   * @param maxRequestBodySize a map specifying the maximum size in bytes for the HTTP request body by Content-Type
    * @return This.
    */
-  default T withMaxRequestBodySize(int maxRequestBodySize) {
+  default T withMaxRequestBodySize(Map<String, Integer> maxRequestBodySize) {
     configuration().withMaxRequestBodySize(maxRequestBodySize);
     return (T) this;
   }

src/main/java/io/fusionauth/http/server/HTTPServerConfiguration.java

@@ -18,7 +18,9 @@
 import java.nio.file.Path;
 import java.time.Duration;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import io.fusionauth.http.io.MultipartConfiguration;
@@ -55,11 +57,11 @@ public class HTTPServerConfiguration implements Configurable<HTTPServerConfigura
 
   private int maxBytesToDrain = 256 * 1024; // 256 Kilobytes
 
-  private int maxFormDataSize = 10 * 1024 * 1024; // 10 Megabytes, ideally same as MultipartConfiguration.maxRequestSize default
-
   private int maxPendingSocketConnections = 250;
 
-  private int maxRequestBodySize = 128 * 1024 * 1024; // 128 Megabytes, must be equal to or larger than maxFormDataSize, and MultipartConfiguration.maxRequestSize
+  private Map<String, Integer> maxRequestBodySize = Map.of(
+      "*", 128 * 1024 * 1024,                                   // 128 Megabytes
+      "application/x-www-form-urlencoded", 10 * 1024 * 1024);   // 10 Megabytes
 
   private int maxRequestHeaderSize = 128 * 1024; // 128 Kilobytes
 
@@ -187,14 +189,6 @@ public int getMaxBytesToDrain() {
     return maxBytesToDrain;
   }
 
-  /**
-   * @return the maximum size of the HTTP request body in bytes when the HTTP server process a Content-Type of
-   *     application/x-www-form-urlencoded. Defaults to 10 Megabytes.
-   */
-  public int getMaxFormDataSize() {
-    return maxFormDataSize;
-  }
-
   /**
    * The maximum number of pending socket connections per HTTP listener.
    * <p>
@@ -209,14 +203,16 @@ public int getMaxPendingSocketConnections() {
   }
 
   /**
-   * The maximum size in bytes of the HTTP request body. This configuration excludes the size of the HTTP request header.
+   * The map that specifies the maximum size in bytes of the HTTP request body by Content-Type. This configuration excludes the size of the
+   * HTTP request header.
    * <p>
-   * This configuration will affect all requests regardless of Content-Type and is intended to be a fail-safe for unexpected large
-   * requests.
+   * The returned map is keyed by Content-Type, and willy contain a default value identified by '*', and may optionally return content type
+   * values with a wild card '*' as the subtype.
    *
-   * @return the maximum size in bytes of the HTTP request body.  Defaults to 128 Megabytes.
+   * @return the map keyed by Content-Type indicating the maximum size in bytes of the HTTP request body.  Defaults to 128 Megabytes as a
+   *     default, and 10 Megabytes for application/x-www-form-urlencoded.
    */
-  public int getMaxRequestBodySize() {
+  public Map<String, Integer> getMaxRequestBodySize() {
     return maxRequestBodySize;
   }
 
@@ -463,19 +459,6 @@ public HTTPServerConfiguration withLoggerFactory(LoggerFactory loggerFactory) {
     return this;
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public HTTPServerConfiguration withMaxFormDataSize(int maxFormDataSize) {
-    if (maxFormDataSize != -1 && maxFormDataSize <= 0) {
-      throw new IllegalArgumentException("The maximum form data size must be greater than 0. Set to -1 to disable this limitation.");
-    }
-
-    this.maxFormDataSize = maxFormDataSize;
-    return this;
-  }
-
   /**
    * {@inheritDoc}
    */
@@ -493,9 +476,22 @@ public HTTPServerConfiguration withMaxPendingSocketConnections(int maxPendingSoc
    * {@inheritDoc}
    */
   @Override
-  public HTTPServerConfiguration withMaxRequestBodySize(int maxRequestBodySize) {
-    if (maxRequestBodySize != -1 && maxRequestBodySize <= 0) {
-      throw new IllegalArgumentException("The maximum request body size must be greater than 0. Set to -1 to disable this limitation.");
+  public HTTPServerConfiguration withMaxRequestBodySize(Map<String, Integer> maxRequestBodySize) {
+    Objects.requireNonNull(maxRequestBodySize, "You cannot set the maximum request body size map to null");
+    for (String contentType : maxRequestBodySize.keySet()) {
+      Objects.requireNonNull(contentType, "You cannot specify a null value for content type");
+      Integer maxSize = maxRequestBodySize.get(contentType);
+      Objects.requireNonNull(maxSize, "You may not specify a null value for the maximum request body size");
+      if (maxSize != -1 && maxSize <= 0) {
+        throw new IllegalArgumentException("The maximum request body size must be greater than 0 for [" + contentType + "]. Set to -1 to disable this limitation.");
+      }
+    }
+
+    // Keep existing default if one was not provided.
+    if (!maxRequestBodySize.containsKey("*")) {
+      Map<String, Integer> copy = new HashMap<>(maxRequestBodySize);
+      copy.put("*", maxRequestBodySize.get("*"));
+      maxRequestBodySize = copy;
     }
 
     this.maxRequestBodySize = maxRequestBodySize;
@@ -593,8 +589,6 @@ public HTTPServerConfiguration withMultipartBufferSize(int multipartBufferSize)
     return this;
   }
 
-  // TODO : Review coments for constraints for multi-part, etc.
-
   @Override
   public HTTPServerConfiguration withMultipartConfiguration(MultipartConfiguration multipartStreamConfiguration) {
     Objects.requireNonNull(multipartStreamConfiguration, "You cannot set the multipart stream configuration to null");

src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java

@@ -25,7 +25,6 @@
 import io.fusionauth.http.HTTPProcessingException;
 import io.fusionauth.http.HTTPValues;
 import io.fusionauth.http.HTTPValues.Connections;
-import io.fusionauth.http.HTTPValues.ContentTypes;
 import io.fusionauth.http.HTTPValues.Headers;
 import io.fusionauth.http.HTTPValues.Protocols;
 import io.fusionauth.http.ParseException;
@@ -145,7 +144,7 @@ public void run() {
           instrumenter.acceptedRequest();
         }
 
-        int maximumContentLength = getMaximumContentLength(request);
+        int maximumContentLength = HTTPTools.getMaxRequestBodySize(request.getContentType(), configuration.getMaxRequestBodySize());
         httpInputStream = new HTTPInputStream(configuration, request, inputStream, maximumContentLength);
         request.setInputStream(httpInputStream);
 
@@ -332,19 +331,6 @@ private void closeSocketOnly(CloseSocketReason reason) {
     }
   }
 
-  private int getMaximumContentLength(HTTPRequest request) {
-    var maximumContentLength = -1;
-    if (ContentTypes.Form.equalsIgnoreCase(request.getContentType())) {
-      maximumContentLength = configuration.getMaxFormDataSize();
-    }
-
-    if (maximumContentLength == -1) {
-      maximumContentLength = configuration.getMaxRequestBodySize();
-    }
-
-    return maximumContentLength;
-  }
-
   private boolean handleExpectContinue(HTTPRequest request) throws IOException {
     var expectResponse = new HTTPResponse();
     configuration.getExpectValidator().validate(request, expectResponse);

src/main/java/io/fusionauth/http/util/HTTPTools.java

@@ -45,6 +45,37 @@
 public final class HTTPTools {
   private static Logger logger;
 
+  /**
+   * Return the maximum request body size for the requested content type.
+   *
+   * @param contentType        the content-type of the request
+   * @param maxRequestBodySize the maximum request size configuration
+   * @return the maximum request size, or -1 if no limit should be enforced.
+   */
+  public static int getMaxRequestBodySize(String contentType, Map<String, Integer> maxRequestBodySize) {
+    if (contentType == null) {
+      return maxRequestBodySize.get("*");
+    }
+
+    // Exact match
+    Integer maximumSize = maxRequestBodySize.get(contentType);
+    if (maximumSize != null) {
+      return maximumSize;
+    }
+
+    // Ignore subtype by replacing it with '*'. RFC 1341 says a subtype is required which means each Content-Type must contain a /.
+    // - But be more defensive, and account for a case where no subtype has been defined.
+    int index = contentType.indexOf('/');
+    if (index != -1) {
+      maximumSize = maxRequestBodySize.get(contentType.substring(0, index) + "/*");
+    }
+
+    // RFC 1341 indicates subtypes cannot be nested. So if we do not yet have a match, use the default key '*'.
+    return maximumSize != null
+        ? maximumSize
+        : maxRequestBodySize.get("*");
+  }
+
   /**
    * Statically sets up the logger, mostly for trace logging.
    *
@@ -343,7 +374,7 @@ public static void parseRequestPreamble(PushbackInputStream inputStream, int max
 
       // index is the number of bytes we processed as part of the preamble
       premableLength += index;
-      if (maxRequestHeaderSize !=-1 && premableLength > maxRequestHeaderSize) {
+      if (maxRequestHeaderSize != -1 && premableLength > maxRequestHeaderSize) {
         throw new RequestHeadersTooLargeException(maxRequestHeaderSize, "The maximum size of the request header has been exceeded. The maximum size is [" + maxRequestHeaderSize + "] bytes.");
       }
     }

src/test/java/io/fusionauth/http/FormDataTest.java

@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.function.Consumer;
 
+import io.fusionauth.http.HTTPValues.ContentTypes;
 import io.fusionauth.http.HTTPValues.Headers;
 import io.fusionauth.http.server.HTTPHandler;
 import io.fusionauth.http.server.HTTPServer;
@@ -60,7 +61,7 @@ public void post_server_configuration_max_form_data(String scheme, boolean chunk
         // Account for the equals size and a separator of & except for the first value
         // - This should mean we have just exactly the right size of configuration for this request body
         // Config is [180,223]
-        .withConfiguration(config -> config.withMaxFormDataSize((4096 * 10) + (4096 * 32) + (4096 * 2) - 1))
+        .withConfiguration(config -> config.withMaxRequestBodySize(Map.of(ContentTypes.Form, (4096 * 10) + (4096 * 32) + (4096 * 2) - 1)))
         .expectResponse("""
             HTTP/1.1 200 \r
             connection: keep-alive\r
@@ -76,7 +77,7 @@ public void post_server_configuration_max_form_data(String scheme, boolean chunk
         .withBodyParameterCount(42 * 1024)  // 43,008
         .withBodyParameterSize(128)
         // 4k * 33 > 128k
-        .withConfiguration(config -> config.withMaxFormDataSize(128 * 1024))
+        .withConfiguration(config -> config.withMaxRequestBodySize(Map.of(ContentTypes.Form, 128 * 1024)))
         .expectResponse("""
             HTTP/1.1 413 \r
             connection: close\r
@@ -88,10 +89,10 @@ public void post_server_configuration_max_form_data(String scheme, boolean chunk
     // Large, but max size has been disabled
     withScheme(scheme)
         .withChunked(chunked)
-        .withBodyParameterCount(42 * 1024)  // 131,072, 131,072
+        .withBodyParameterCount(42 * 1024)  // 43,008
         .withBodyParameterSize(128)
         // Disable the limit
-        .withConfiguration(config -> config.withMaxFormDataSize(-1))
+        .withConfiguration(config -> config.withMaxRequestBodySize(Map.of(ContentTypes.Form, -1)))
         .expectResponse("""
             HTTP/1.1 200 \r
             connection: keep-alive\r
@@ -100,6 +101,21 @@ public void post_server_configuration_max_form_data(String scheme, boolean chunk
             \r
             {"version":"42"}""")
         .expectNoExceptionOnWrite();
+
+    // Large, enforce using default w/out a specific configuration for application/x-www-form-urlencoded
+    withScheme(scheme)
+        .withChunked(chunked)
+        .withBodyParameterCount(42 * 1024)  // 131,072, 131,072
+        .withBodyParameterSize(128)
+        // Disable the limit
+        .withConfiguration(config -> config.withMaxRequestBodySize(Map.of("*", 128 * 1024)))
+        .expectResponse("""
+            HTTP/1.1 413 \r
+            connection: close\r
+            content-length: 0\r
+            \r
+            """)
+        .expectExceptionOnWrite(SocketException.class);
   }
 
   @Test(dataProvider = "schemes")
@@ -313,7 +329,7 @@ public Builder withChunked(boolean chunked) {
       return this;
     }
 
-    public Builder withConfiguration(Consumer<HTTPServerConfiguration> configuration) throws Exception {
+    public Builder withConfiguration(Consumer<HTTPServerConfiguration> configuration) {
       this.configuration = configuration;
       return this;
     }

src/test/java/io/fusionauth/http/util/HTTPToolsTest.java

@@ -42,6 +42,29 @@
  */
 @Test
 public class HTTPToolsTest {
+  @Test
+  public void getMaxRequestBodySize() {
+    var configuration = Map.of(
+        "*", 1,
+        "application/*", 2,
+        "application/json", 3,
+        "application/x-www-form-urlencoded", 4,
+        "multipart/form-data", 5,
+        "text/*", 6,
+        "text/html", 7
+    );
+
+    assertMaxConfiguredSize(null, 1, configuration);
+    assertMaxConfiguredSize("application/json", 3, configuration);
+    assertMaxConfiguredSize("application/json-patch+json", 2, configuration);
+    assertMaxConfiguredSize("application/octet-stream", 2, configuration);
+    assertMaxConfiguredSize("application/pdf", 2, configuration);
+    assertMaxConfiguredSize("application/x-www-form-urlencoded", 4, configuration);
+    assertMaxConfiguredSize("multipart/form-data", 5, configuration);
+    assertMaxConfiguredSize("text/css", 6, configuration);
+    assertMaxConfiguredSize("text/html", 7, configuration);
+  }
+
   @Test
   public void parseEncodedData() {
     // Happy path
@@ -240,6 +263,10 @@ private void assertHexValue(String s, String expected, Charset charset) {
     assertEquals(hex(s.getBytes(charset)), trimmed);
   }
 
+  private void assertMaxConfiguredSize(String contentType, int maximumSize, Map<String, Integer> maxRequestBodySize) {
+    assertEquals(maximumSize, HTTPTools.getMaxRequestBodySize(contentType, maxRequestBodySize));
+  }
+
   private String hex(byte[] bytes) {
     List<String> result = new ArrayList<>();
     for (byte b : bytes) {