global model testing implemented

Author: gaborvecsei

.gitignore

@@ -1,2 +1,3 @@
 .idea/
 __pycache__
+*.json

fed_learn/fed_server.py

@@ -18,32 +18,42 @@ def __init__(self, model_fn: Callable,
         # Initialize the global model's weights
         self.model_fn = model_fn
         model = self.model_fn()
+        self.global_test_metrics_dict = {k: [] for k in model.metrics_names}
         self.global_model_weights = model.get_weights()
         fed_learn.get_rid_of_the_models(model)
 
-        self.global_losses = []
+        self.global_train_losses = []
         self.epoch_losses = []
 
-        (x_train, y_train), _ = datasets.cifar10.load_data()
+        (x_train, y_train), (x_test, y_test) = datasets.cifar10.load_data()
 
         if only_debugging:
             # TODO: remove me
             x_train = x_train[:100]
             y_train = y_train[:100]
+            x_test = x_test[:100]
+            y_test = y_test[:100]
 
         # TODO: separate preprocessor for the data transformations
         y_train = utils.to_categorical(y_train, len(np.unique(y_train)))
         x_train = x_train.astype(np.float32)
         x_train /= 255.0
 
+        y_test = utils.to_categorical(y_test, len(np.unique(y_test)))
+        x_test = x_test.astype(np.float32)
+        x_test /= 255.0
+
         self.x_train = x_train
         self.y_train = y_train
+        self.x_test = x_test
+        self.y_test = y_test
 
         self.client_data_indices = None
         self.clients = []
         self.client_model_weights = []
 
         # Training parameters used by the clients
+        # TODO: this should be configurable
         self.train_dict = {"batch_size": 32,
                            "epochs": 5,
                            "verbose": 1,
@@ -96,3 +106,16 @@ def summarize_weights(self):
 
     def get_client_train_param_dict(self):
         return self.train_dict
+
+    def test_global_model(self):
+        model = self.model_fn()
+        fed_learn.models.set_model_weights(model, self.global_model_weights)
+        results = model.evaluate(self.x_test, self.y_test, batch_size=32, verbose=1)
+
+        results_dict = dict(zip(model.metrics_names, results))
+        for metric_name, value in results_dict.items():
+            self.global_test_metrics_dict[metric_name].append(value)
+
+        fed_learn.get_rid_of_the_models(model)
+
+        return results_dict

fl.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 
 import numpy as np
 
@@ -42,9 +43,16 @@ def model_fn():
     server.summarize_weights()
 
     epoch_mean_loss = np.mean(server.epoch_losses)
-    server.global_losses.append(epoch_mean_loss)
-    print("Loss (mean): {0}".format(server.global_losses[-1]))
+    server.global_train_losses.append(epoch_mean_loss)
+    print("Loss (client mean): {0}".format(server.global_train_losses[-1]))
 
-    print("-" * 30)
+    global_test_results = server.test_global_model()
+    for metric_name, value in global_test_results.items():
+        print("Global test|")
+        print("_" * 10)
+        print("{0}: {1}".format(metric_name, value))
 
-    # TODO: test the base model with the aggregated weights
+    print("_" * 30)
+
+with open("fed_learn_global_test_results.json", 'w') as f:
+    json.dump(server.global_test_metrics_dict, f)