Store raw responseBody and decompress when needed

src/core/build.gradle.kts

@@ -114,6 +114,7 @@ dependencies {
         isTransitive = false
     }
     implementation("org.apache.xmlgraphics:xmlgraphics-commons")
+    implementation("org.brotli:dec")
     implementation("org.freemarker:freemarker")
     implementation("org.jodd:jodd-core")
     implementation("org.jodd:jodd-props")

src/core/src/main/java/org/apache/jmeter/samplers/SampleResult.java

@@ -17,6 +17,10 @@
 
 package org.apache.jmeter.samplers;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
@@ -25,9 +29,13 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
 
 import org.apache.jmeter.assertions.AssertionResult;
 import org.apache.jmeter.gui.Searchable;
@@ -160,6 +168,8 @@ public class SampleResult implements Serializable, Cloneable, Searchable {
 
     private byte[] responseData = EMPTY_BA;
 
+    private String contentEncoding; // Stores gzip/deflate encoding if response is compressed
+
     private String responseCode = "";// Never return null
 
     private String label = "";// Never return null
@@ -217,7 +227,7 @@ public class SampleResult implements Serializable, Cloneable, Searchable {
 
     // TODO do contentType and/or dataEncoding belong in HTTPSampleResult instead?
     private String dataEncoding;// (is this really the character set?) e.g.
-                                // ISO-8895-1, UTF-8
+    // ISO-8895-1, UTF-8
 
     private String contentType = ""; // e.g. text/html; charset=utf-8
 
@@ -791,6 +801,27 @@ public void setResponseData(final String response, final String encoding) {
      * @return the responseData value (cannot be null)
      */
     public byte[] getResponseData() {
+        if (responseData == null) {
+            return EMPTY_BA;
+        }
+        if (contentEncoding != null && responseData.length > 0) {
+            try {
+                switch (contentEncoding.toLowerCase(Locale.ROOT)) {
+                    case "gzip":
+                        return decompressGzip(responseData);
+                    case "x-gzip":
+                        return decompressGzip(responseData);
+                    case "deflate":
+                        return decompressDeflate(responseData);
+                    case "br":
+                        return decompressBrotli(responseData);
+                    default:
+                        return responseData;
+                }
+            } catch (IOException e) {
+                log.warn("Failed to decompress response data", e);
+            }
+        }
         return responseData;
     }
 
@@ -802,12 +833,12 @@ public byte[] getResponseData() {
     public String getResponseDataAsString() {
         try {
             if(responseDataAsString == null) {
-                responseDataAsString= new String(responseData,getDataEncodingWithDefault());
+                responseDataAsString= new String(getResponseData(),getDataEncodingWithDefault());
             }
             return responseDataAsString;
         } catch (UnsupportedEncodingException e) {
             log.warn("Using platform default as {} caused {}", getDataEncodingWithDefault(), e.getLocalizedMessage());
-            return new String(responseData,Charset.defaultCharset()); // N.B. default charset is used deliberately here
+            return new String(getResponseData(),Charset.defaultCharset()); // N.B. default charset is used deliberately here
         }
     }
 
@@ -1665,4 +1696,63 @@ public TestLogicalAction getTestLogicalAction() {
     public void setTestLogicalAction(TestLogicalAction testLogicalAction) {
         this.testLogicalAction = testLogicalAction;
     }
+
+    /**
+     * Sets the response data and its compression encoding.
+     * @param data The response data
+     * @param encoding The content encoding (e.g. gzip, deflate)
+     */
+    public void setResponseData(byte[] data, String encoding) {
+        responseData = data == null ? EMPTY_BA : data;
+        contentEncoding = encoding;
+        responseDataAsString = null;
+    }
+
+    private static byte[] decompressGzip(byte[] in) throws IOException {
+        try (GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(in));
+             ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            byte[] buf = new byte[8192];
+            int len;
+            while ((len = gis.read(buf)) > 0) {
+                out.write(buf, 0, len);
+            }
+            return out.toByteArray();
+        }
+    }
+
+    private static byte[] decompressDeflate(byte[] in) throws IOException {
+        // Try with ZLIB wrapper first
+        try {
+            return decompressWithInflater(in, false);
+        } catch (IOException e) {
+            // If that fails, try with NO_WRAP for raw DEFLATE
+            return decompressWithInflater(in, true);
+        }
+    }
+
+    private static byte[] decompressWithInflater(byte[] in, boolean nowrap) throws IOException {
+        try (InflaterInputStream iis = new InflaterInputStream(
+                new ByteArrayInputStream(in),
+                new Inflater(nowrap));
+             ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            byte[] buf = new byte[8192];
+            int len;
+            while ((len = iis.read(buf)) > 0) {
+                out.write(buf, 0, len);
+            }
+            return out.toByteArray();
+        }
+    }
+
+    private static byte[] decompressBrotli(byte[] in) throws IOException {
+        try (InputStream bis = new org.brotli.dec.BrotliInputStream(new ByteArrayInputStream(in));
+             ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+            byte[] buf = new byte[8192];
+            int len;
+            while ((len = bis.read(buf)) > 0) {
+                out.write(buf, 0, len);
+            }
+            return out.toByteArray();
+        }
+    }
 }

src/protocol/http/build.gradle.kts

@@ -73,7 +73,6 @@ dependencies {
     implementation("dnsjava:dnsjava")
     implementation("org.apache.httpcomponents:httpmime")
     implementation("org.apache.httpcomponents:httpcore")
-    implementation("org.brotli:dec")
     implementation("com.miglayout:miglayout-swing")
     implementation("com.fasterxml.jackson.core:jackson-core")
     implementation("com.fasterxml.jackson.core:jackson-databind")

src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPHC4Impl.java

@@ -57,7 +57,6 @@
 import org.apache.http.HttpRequest;
 import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.HttpResponse;
-import org.apache.http.HttpResponseInterceptor;
 import org.apache.http.NameValuePair;
 import org.apache.http.StatusLine;
 import org.apache.http.auth.AuthSchemeProvider;
@@ -72,7 +71,6 @@
 import org.apache.http.client.config.AuthSchemes;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.entity.InputStreamFactory;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
@@ -86,7 +84,6 @@
 import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.protocol.ResponseContentEncoding;
 import org.apache.http.config.Lookup;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
@@ -147,8 +144,6 @@
 import org.apache.jmeter.protocol.http.control.DynamicKerberosSchemeFactory;
 import org.apache.jmeter.protocol.http.control.DynamicSPNegoSchemeFactory;
 import org.apache.jmeter.protocol.http.control.HeaderManager;
-import org.apache.jmeter.protocol.http.sampler.hc.LaxDeflateInputStream;
-import org.apache.jmeter.protocol.http.sampler.hc.LaxGZIPInputStream;
 import org.apache.jmeter.protocol.http.sampler.hc.LazyLayeredConnectionSocketFactory;
 import org.apache.jmeter.protocol.http.util.ConversionUtils;
 import org.apache.jmeter.protocol.http.util.HTTPArgument;
@@ -166,7 +161,6 @@
 import org.apache.jmeter.util.JsseSSLManager;
 import org.apache.jmeter.util.SSLManager;
 import org.apache.jorphan.util.JOrphanUtils;
-import org.brotli.dec.BrotliInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -195,20 +189,8 @@ public class HTTPHC4Impl extends HTTPHCAbstractImpl {
 
     private static final boolean DISABLE_DEFAULT_UA = JMeterUtils.getPropDefault("httpclient4.default_user_agent_disabled", false);
 
-    private static final boolean GZIP_RELAX_MODE = JMeterUtils.getPropDefault("httpclient4.gzip_relax_mode", false);
-
-    private static final boolean DEFLATE_RELAX_MODE = JMeterUtils.getPropDefault("httpclient4.deflate_relax_mode", false);
-
     private static final Logger log = LoggerFactory.getLogger(HTTPHC4Impl.class);
 
-    private static final InputStreamFactory GZIP =
-            instream -> new LaxGZIPInputStream(instream, GZIP_RELAX_MODE);
-
-    private static final InputStreamFactory DEFLATE =
-            instream -> new LaxDeflateInputStream(instream, DEFLATE_RELAX_MODE);
-
-    private static final InputStreamFactory BROTLI = BrotliInputStream::new;
-
     private static final class ManagedCredentialsProvider implements CredentialsProvider {
         private final AuthManager authManager;
         private final Credentials proxyCredentials;
@@ -472,55 +454,6 @@ protected HttpResponse doSendRequest(
         }
     };
 
-    private static final String[] HEADERS_TO_SAVE = new String[]{
-                    "content-length",
-                    "content-encoding",
-                    "content-md5"
-            };
-
-    /**
-     * Custom implementation that backups headers related to Compressed responses
-     * that HC core {@link ResponseContentEncoding} removes after uncompressing
-     * See Bug 59401
-     */
-    @SuppressWarnings("UnnecessaryAnonymousClass")
-    private static final HttpResponseInterceptor RESPONSE_CONTENT_ENCODING = new ResponseContentEncoding(createLookupRegistry()) {
-        @Override
-        public void process(HttpResponse response, HttpContext context)
-                throws HttpException, IOException {
-            ArrayList<Header[]> headersToSave = null;
-
-            final HttpEntity entity = response.getEntity();
-            final HttpClientContext clientContext = HttpClientContext.adapt(context);
-            final RequestConfig requestConfig = clientContext.getRequestConfig();
-            // store the headers if necessary
-            if (requestConfig.isContentCompressionEnabled() && entity != null && entity.getContentLength() != 0) {
-                final Header ceheader = entity.getContentEncoding();
-                if (ceheader != null) {
-                    headersToSave = new ArrayList<>(3);
-                    for(String name : HEADERS_TO_SAVE) {
-                        Header[] hdr = response.getHeaders(name); // empty if none
-                        headersToSave.add(hdr);
-                    }
-                }
-            }
-
-            // Now invoke original parent code
-            super.process(response, clientContext);
-            // Should this be in a finally ?
-            if(headersToSave != null) {
-                for (Header[] headers : headersToSave) {
-                    for (Header headerToRestore : headers) {
-                        if (response.containsHeader(headerToRestore.getName())) {
-                            break;
-                        }
-                        response.addHeader(headerToRestore);
-                    }
-                }
-            }
-        }
-    };
-
     /**
      * 1 HttpClient instance per combination of (HttpClient,HttpClientKey)
      */
@@ -558,19 +491,6 @@ protected HTTPHC4Impl(HTTPSamplerBase testElement) {
         super(testElement);
     }
 
-    /**
-     * Customize to plug Brotli
-     * @return {@link Lookup}
-     */
-    private static Lookup<InputStreamFactory> createLookupRegistry() {
-        return
-                RegistryBuilder.<InputStreamFactory>create()
-                .register("br", BROTLI)
-                .register("gzip", GZIP)
-                .register("x-gzip", GZIP)
-                .register("deflate", DEFLATE).build();
-    }
-
     /**
      * Implementation that allows GET method to have a body
      */
@@ -675,7 +595,12 @@ protected HTTPSampleResult sample(URL url, String method,
             }
             HttpEntity entity = httpResponse.getEntity();
             if (entity != null) {
-                res.setResponseData(readResponse(res, entity.getContent(), entity.getContentLength()));
+                Header contentEncodingHeader = entity.getContentEncoding();
+                if (contentEncodingHeader != null) {
+                    res.setResponseData(EntityUtils.toByteArray(entity), contentEncodingHeader.getValue());
+                } else {
+                    res.setResponseData(EntityUtils.toByteArray(entity));
+                }
             }
 
             res.sampleEnd(); // Done with the sampling proper.
@@ -1157,7 +1082,7 @@ private MutableTriple<CloseableHttpClient, AuthState, PoolingHttpClientConnectio
                 }
                 builder.setDefaultCredentialsProvider(credsProvider);
             }
-            builder.disableContentCompression().addInterceptorLast(RESPONSE_CONTENT_ENCODING);
+            builder.disableContentCompression(); // Disable automatic decompression
             if(BASIC_AUTH_PREEMPTIVE) {
                 builder.addInterceptorFirst(PREEMPTIVE_AUTH_INTERCEPTOR);
             } else {

src/protocol/http/src/main/java/org/apache/jmeter/protocol/http/sampler/HTTPJavaImpl.java

@@ -31,7 +31,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.function.Predicate;
-import java.util.zip.GZIPInputStream;
 
 import org.apache.commons.io.input.CountingInputStream;
 import org.apache.jmeter.protocol.http.control.AuthManager;
@@ -240,15 +239,11 @@ protected byte[] readResponse(HttpURLConnection conn, SampleResult res) throws I
         }
 
         // works OK even if ContentEncoding is null
-        boolean gzipped = HTTPConstants.ENCODING_GZIP.equals(conn.getContentEncoding());
+        String contentEncoding = conn.getContentEncoding();
         CountingInputStream instream = null;
         try {
             instream = new CountingInputStream(conn.getInputStream());
-            if (gzipped) {
-                in = new GZIPInputStream(instream);
-            } else {
-                in = instream;
-            }
+            in = instream;
         } catch (IOException e) {
             if (! (e.getCause() instanceof FileNotFoundException))
             {
@@ -276,28 +271,15 @@ protected byte[] readResponse(HttpURLConnection conn, SampleResult res) throws I
                 log.info("Error Response Code: {}", conn.getResponseCode());
             }
 
-            if (gzipped) {
-                in = new GZIPInputStream(errorStream);
-            } else {
-                in = errorStream;
-            }
-        } catch (Exception e) {
-            log.error("readResponse: {}", e.toString());
-            Throwable cause = e.getCause();
-            if (cause != null){
-                log.error("Cause: {}", cause.toString());
-                if(cause instanceof Error) {
-                    throw (Error)cause;
-                }
-            }
-            in = conn.getErrorStream();
+            in = errorStream;
         }
         // N.B. this closes 'in'
         byte[] responseData = readResponse(res, in, contentLength);
         if (instream != null) {
             res.setBodySize(instream.getByteCount());
             instream.close();
         }
+        res.setResponseData(responseData, contentEncoding);
         return responseData;
     }