Ensure reauthentication attempts aren't blocked after unexpected error (#44)

Motivation

A resiliency bug was encountered where the client fails to
reauthenticate/recover following an abnormal response from the server to
the authenticate RPC request.

Modification

Ensure the status stored/propagated from a failed authenticate RPC will
always be interpreted as authentication-related, so that the existing
logic will properly re-attempt authentication in all cases.

Result

Client properly recovers in rare server failure scenario

Author: njhill

src/main/java/com/ibm/etcd/client/EtcdClient.java

@@ -602,7 +602,7 @@ public boolean isClosed() {
     };
 
     private Status lastAuthFailStatus;
-    private long lastAuthFailTime;
+    private long authFailRetryTime;
 
     protected static boolean reauthRequired(Throwable error) {
         Status status = Status.fromThrowable(error);
@@ -616,8 +616,7 @@ protected static boolean reauthRequired(Throwable error) {
 
     // This is only called in a synchronized context
     private CallCredentials refreshCredentials(Throwable trigger) {
-        long lastAuthFailure = System.currentTimeMillis() - lastAuthFailTime;
-        if (lastAuthFailure < 15_000L) {
+        if (System.currentTimeMillis() < authFailRetryTime) {
             // Rate-limit how often re-auth attempts are made,
             // return last auth failure in the meantime
             return new CallCredentials() {
@@ -645,9 +644,24 @@ public void applyRequestMetadata(RequestInfo requestInfo,
                     try {
                         applier.apply(futureTokenHeader.get());
                     } catch (ExecutionException | InterruptedException ee) { // (IE won't be thrown)
-                        Status failStatus = Status.fromThrowable(ee.getCause());
+                        Throwable failure = ee.getCause();
+                        Status failStatus;
+                        if (!reauthRequired(failure)) {
+                            // The authenticate RPC failed with an unexpected error (non auth-related)
+                            // Ensure our top-level status remains UNAUTHENTICATED or else higher
+                            // level logic will not recognize that another reauth attempt is required.
+                            failStatus = Status.UNAUTHENTICATED
+                                    .withDescription("(Re)authentication RPC failed")
+                                    .withCause(failure);
+                            authFailRetryTime = System.currentTimeMillis() + 5_000L;
+                        } else {
+                            failStatus = Status.fromThrowable(failure);
+                            // If this was a real auth failure, postpone further attempts a bit longer
+                            authFailRetryTime = System.currentTimeMillis() + 15_000L;
+                        }
                         lastAuthFailStatus = failStatus;
-                        lastAuthFailTime = System.currentTimeMillis();
+                        // Augment with the RPC failure that triggered the re-authentication,
+                        // if applicable
                         if (trigger != null) {
                             if (failStatus.getCause() != null) {
                                 failStatus.getCause().addSuppressed(trigger);