[fix] read_nonblock(exception: false) throwing on TLS 1.3

propagate exception flag through read() and readAndUnwrap()

Author: kares

src/main/java/org/jruby/ext/openssl/SSLSocket.java

@@ -539,8 +539,18 @@ public void wakeup() {
         }
     }
 
-    private static final int READ_WOULD_BLOCK_RESULT = Integer.MIN_VALUE + 1;
-    private static final int WRITE_WOULD_BLOCK_RESULT = Integer.MIN_VALUE + 2;
+    // Legitimate return values are -1 (EOF) and >= 0 (byte counts), so any value < -1 is safely in sentinel territory.
+    private static final int READ_WOULD_BLOCK_RESULT  = -2;
+    private static final int WRITE_WOULD_BLOCK_RESULT = -3;
+
+    private static boolean isWouldBlockResult(final int result) {
+        return result < -1;
+    }
+
+    private RubySymbol wouldBlockSymbol(final int result) {
+        assert isWouldBlockResult(result) : "unexpected result: " + result;
+        return getRuntime().newSymbol(result == READ_WOULD_BLOCK_RESULT ? "wait_readable" : "wait_writable");
+    }
 
     private static void readWouldBlock(final Ruby runtime, final boolean exception, final int[] result) {
         if ( exception ) throw newSSLErrorWaitReadable(runtime, "read would block");
@@ -552,10 +562,6 @@ private static void writeWouldBlock(final Ruby runtime, final boolean exception,
         result[0] = WRITE_WOULD_BLOCK_RESULT;
     }
 
-    private void doHandshake(final boolean blocking) throws IOException {
-        doHandshake(blocking, true);
-    }
-
     // might return :wait_readable | :wait_writable in case (true, false)
     private IRubyObject doHandshake(final boolean blocking, final boolean exception) throws IOException {
         while (true) {
@@ -577,7 +583,11 @@ private IRubyObject doHandshake(final boolean blocking, final boolean exception)
                 doTasks();
                 break;
             case NEED_UNWRAP:
-                if (readAndUnwrap(blocking) == -1 && handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED) {
+                int unwrapResult = readAndUnwrap(blocking, exception);
+                if (isWouldBlockResult(unwrapResult)) {
+                    return wouldBlockSymbol(unwrapResult);
+                }
+                if (unwrapResult == -1 && handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED) {
                     throw new SSLHandshakeException("Socket closed");
                 }
                 // during initialHandshake, calling readAndUnwrap that results UNDERFLOW does not mean writable.
@@ -703,12 +713,16 @@ public int write(ByteBuffer src, boolean blocking) throws SSLException, IOExcept
     }
 
     public int read(final ByteBuffer dst, final boolean blocking) throws IOException {
+        return read(dst, blocking, true);
+    }
+
+    private int read(final ByteBuffer dst, final boolean blocking, final boolean exception) throws IOException {
         if ( initialHandshake ) return 0;
         if ( engine.isInboundDone() ) return -1;
 
         if ( ! appReadData.hasRemaining() ) {
-            int appBytesProduced = readAndUnwrap(blocking);
-            if (appBytesProduced == -1 || appBytesProduced == 0) {
+            final int appBytesProduced = readAndUnwrap(blocking, exception);
+            if (appBytesProduced == -1 || appBytesProduced == 0 || isWouldBlockResult(appBytesProduced)) {
                 return appBytesProduced;
             }
         }
@@ -718,7 +732,15 @@ public int read(final ByteBuffer dst, final boolean blocking) throws IOException
         return limit;
     }
 
-    private int readAndUnwrap(final boolean blocking) throws IOException {
+    /**
+     * @param blocking whether to block on I/O
+     * @param exception when false, returns {@link #READ_WOULD_BLOCK_RESULT} or
+     *                  {@link #WRITE_WOULD_BLOCK_RESULT} instead of throwing if the
+     *                  post-handshake processing would block
+     * @return application bytes available, -1 on EOF/close, 0 when no app data
+     *         produced, or a WOULD_BLOCK sentinel when would-block with exception=false
+     */
+    private int readAndUnwrap(final boolean blocking, final boolean exception) throws IOException {
         final int bytesRead = socketChannelImpl().read(netReadData);
         if ( bytesRead == -1 ) {
             if ( ! netReadData.hasRemaining() ||
@@ -767,7 +789,11 @@ private int readAndUnwrap(final boolean blocking) throws IOException {
                 handshakeStatus == SSLEngineResult.HandshakeStatus.NEED_TASK ||
                 handshakeStatus == SSLEngineResult.HandshakeStatus.NEED_WRAP ||
                 handshakeStatus == SSLEngineResult.HandshakeStatus.FINISHED ) ) {
-            doHandshake(blocking);
+            IRubyObject ex = doHandshake(blocking, exception);
+            if ( ex != null ) { // :wait_readable | :wait_writable
+                // TODO needs refactoring to avoid Symbol -> int -> Symbol
+                return "wait_writable".equals(ex.asJavaString()) ? WRITE_WOULD_BLOCK_RESULT : READ_WOULD_BLOCK_RESULT;
+            }
         }
         return appReadData.remaining();
     }
@@ -843,12 +869,18 @@ private IRubyObject sysreadImpl(final ThreadContext context, final IRubyObject l
                 if ( engine == null ) {
                     read = socketChannelImpl().read(dst);
                 } else {
-                    read = read(dst, blocking);
+                    read = read(dst, blocking, exception);
                 }
 
-                if ( read == -1 ) {
-                    if ( exception ) throw runtime.newEOFError();
-                    return context.nil;
+                switch ( read ) {
+                    case -1 :
+                        if ( exception ) throw runtime.newEOFError();
+                        return context.nil;
+                    // Post-handshake processing (e.g. TLS 1.3 NewSessionTicket) signaled would-block
+                    case READ_WOULD_BLOCK_RESULT :
+                        return runtime.newSymbol("wait_readable");
+                    case WRITE_WOULD_BLOCK_RESULT :
+                        return runtime.newSymbol("wait_writable");
                 }
 
                 if ( read == 0 && status == SSLEngineResult.Status.BUFFER_UNDERFLOW ) {