TUN-5973: Add backoff for non-recoverable errors as well

Errors that are non-recoverable can lead to one of two things happening:
1. That connection lying dead and cloudflared not retrying to make that
   connection.
2. cloudflared resolving to a different edge addr to retry connection.

We should subject these errors to a backoff as well. This will result in
us introducing a backoff for 1. When we are going to let the connection
become stale anyway and 2. When we are about to try a different edge
addr.

Author: sudarshan-reddy

edgediscovery/edgediscovery.go

@@ -114,7 +114,8 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetDifferentAddr: Giving connection its new address")
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address: %v from the address list: %v",
+		addr, ed.regions.AvailableAddrs())
 	return addr, nil
 }
 

supervisor/tunnel.go

@@ -171,24 +171,26 @@ func ServeTunnelLoop(
 			protocolFallback.protocol,
 			gracefulShutdownC,
 		)
-		if !recoverable {
-			return err
-		}
-
-		config.Observer.SendReconnect(connIndex)
 
-		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
-		if !ok {
-			return err
+		if recoverable {
+			duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
+			if !ok {
+				return err
+			}
+			config.Observer.SendReconnect(connIndex)
+			connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
 		}
-		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
 
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-gracefulShutdownC:
 			return nil
 		case <-protocolFallback.BackoffTimer():
+			if !recoverable {
+				return err
+			}
+
 			if !selectNextProtocol(
 				connLog.Logger(),
 				protocolFallback,