Applied mypy + flake8 + ISort for all files (#58)

* Applied mypy + flake8 for all files

* Sorted imports with ISort

* Moved env change to runner

* fixed all mypy errors and added mypy check to CI

* Yet another mypy fixes

* removed E265 ignoring for flake8 job

* Applying comments

Author: Igor Rukhovich

azure-pipelines.yml

@@ -63,5 +63,18 @@ jobs:
   - script: |
       python -m pip install --upgrade pip setuptools
       pip install flake8
-      flake8 --ignore=E265,E722,E402 --max-line-length=90 --count
+      flake8 --max-line-length=100 --count
     displayName: 'PEP 8 check'
+- job: Mypy
+  pool:
+    vmImage: 'ubuntu-20.04'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.7'
+      addToPath: true
+  - script: |
+      python -m pip install --upgrade pip setuptools
+      pip install mypy data-science-types
+      mypy . --ignore-missing-imports
+    displayName: 'mypy check'

bench.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,14 +12,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
 import argparse
-import numpy as np
-import sklearn
-import timeit
 import json
 import sys
+import timeit
+
+import numpy as np
+import sklearn
 
 
 def get_dtype(data):

cuml_bench/__init__.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,17 +12,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import bench
 
+import bench
 from cuml import DBSCAN
 from sklearn.metrics.cluster import davies_bouldin_score
 
+
 parser = argparse.ArgumentParser(description='cuML DBSCAN benchmark')
 parser.add_argument('-e', '--eps', '--epsilon', type=float, default=10.,
                     help='Radius of neighborhood of a point')

cuml_bench/dbscan.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,12 +12,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from typing import Any
+
 import bench
 import cuml
 from cuml.ensemble import RandomForestClassifier
@@ -63,6 +62,7 @@
     params.split_algorithm = 1
 
 params.n_classes = y_train[y_train.columns[0]].nunique()
+clf: Any
 
 
 def fit(X, y):
@@ -80,6 +80,7 @@ def fit(X, y):
 
 
 def predict(X):
+    global clf
     prediction_args = {'predict_model': 'GPU'}
     if int(cuml.__version__.split('.')[1]) <= 14:
         prediction_args.update({'num_classes': params.n_classes})

cuml_bench/df_clsf.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,14 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import bench
+from typing import Any
 
+import bench
 from cuml.ensemble import RandomForestRegressor
 
 parser = argparse.ArgumentParser(description='cuml random forest '
@@ -61,6 +59,7 @@
     params.split_algorithm = 0
 else:
     params.split_algorithm = 1
+regr: Any
 
 
 # Create our random forest regressor
@@ -79,6 +78,7 @@ def fit(X, y):
 
 
 def predict(X):
+    global regr
     return regr.predict(X, predict_model='GPU')
 
 

cuml_bench/df_regr.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,16 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import bench
 
+import bench
 from cuml.linear_model import ElasticNet
 
+
 parser = argparse.ArgumentParser(description='scikit-learn elastic-net regression '
                                              'benchmark')
 parser.add_argument('--no-fit-intercept', dest='fit_intercept', default=True,

cuml_bench/elasticnet.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,17 +12,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import bench
+import warnings
+from typing import Any
 
+import bench
 import numpy as np
 from cuml import KMeans
-import warnings
 from sklearn.metrics.cluster import davies_bouldin_score
 
 warnings.filterwarnings('ignore', category=FutureWarning)
@@ -41,16 +39,18 @@
 # Load and convert generated data
 X_train, X_test, _, _ = bench.load_data(params)
 
+X_init: Any
 if params.filei == 'k-means++':
     X_init = 'k-means++'
 # Load initial centroids from specified path
 elif params.filei is not None:
-    X_init = np.load(params.filei).astype(params.dtype)
-    params.n_clusters = X_init.shape[0]
+    X_init = {k: v.astype(params.dtype) for k, v in np.load(params.filei).items()}
+    if isinstance(X_init, np.ndarray):
+        params.n_clusters = X_init.shape[0]
 # or choose random centroids from training data
 else:
     np.random.seed(params.seed)
-    centroids_idx = np.random.randint(0, X_train.shape[0],
+    centroids_idx = np.random.randint(low=0, high=X_train.shape[0],
                                       size=params.n_clusters)
     if hasattr(X_train, "iloc"):
         X_init = X_train.iloc[centroids_idx].to_pandas().values

cuml_bench/kmeans.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import bench
 from cuml.neighbors import KNeighborsClassifier
 
+
 parser = argparse.ArgumentParser(
     description='cuML kNN classifier benchmark')
 
@@ -77,4 +76,5 @@
                        stages=['training', 'search'], params=params,
                        functions=['knn_clsf.fit', 'knn_clsf.kneighbors'],
                        times=[train_time, predict_time],
+                       accuracies=[], accuracy_type=None,
                        data=[X_train, X_test], alg_instance=knn_clsf)

cuml_bench/knn_clsf.py

@@ -1,4 +1,4 @@
-#===============================================================================
+# ===============================================================================
 # Copyright 2020-2021 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,16 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#===============================================================================
+# ===============================================================================
 
-import sys
-import os
 import argparse
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import bench
 
+import bench
 from cuml.linear_model import Lasso
 
+
 parser = argparse.ArgumentParser(description='scikit-learn lasso regression '
                                              'benchmark')
 parser.add_argument('--no-fit-intercept', dest='fit_intercept', default=False,